{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.3629489603024574,
  "eval_steps": 500,
  "global_step": 10000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 44.075,
      "epoch": 0.0023629489603024575,
      "format_reward": -1.75,
      "grad_norm": 0.179437518119812,
      "image_reward": 0.292385521862242,
      "kl": 0.0005639283277560026,
      "learning_rate": 5e-06,
      "loss": -0.0818,
      "reward": -1.718647839128971,
      "reward_std": 2.0869705460965635,
      "rewards/reward_func": -1.718647839128971,
      "step": 10,
      "toxic_reward": 3.753792663415273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.35,
      "epoch": 0.004725897920604915,
      "format_reward": -1.75,
      "grad_norm": 0.40540918707847595,
      "image_reward": 0.28610331267118455,
      "kl": 0.0006540146190673113,
      "learning_rate": 5e-06,
      "loss": 0.0547,
      "reward": -0.9438592553138733,
      "reward_std": 3.9592867106199265,
      "rewards/reward_func": -0.9438592553138733,
      "step": 20,
      "toxic_reward": 3.622282150387764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.35,
      "epoch": 0.007088846880907372,
      "format_reward": -2.5,
      "grad_norm": 0.3070058524608612,
      "image_reward": 0.29500325620174406,
      "kl": 0.0006122831255197525,
      "learning_rate": 5e-06,
      "loss": 0.0196,
      "reward": -2.2396623373031614,
      "reward_std": 4.928562045097351,
      "rewards/reward_func": -2.2396623373031614,
      "step": 30,
      "toxic_reward": 3.3049886375665665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.475,
      "epoch": 0.00945179584120983,
      "format_reward": -2.25,
      "grad_norm": 0.30812421441078186,
      "image_reward": 0.2784423798322678,
      "kl": 0.0007215021323645487,
      "learning_rate": 5e-06,
      "loss": -0.0374,
      "reward": -1.9919262409210206,
      "reward_std": 3.2468371063470842,
      "rewards/reward_func": -1.9919262409210206,
      "step": 40,
      "toxic_reward": 4.284190082550049
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.0,
      "epoch": 0.011814744801512287,
      "format_reward": -2.25,
      "grad_norm": 0.7593560814857483,
      "image_reward": 0.3014272041618824,
      "kl": 0.0006584389615454711,
      "learning_rate": 5e-06,
      "loss": 0.2018,
      "reward": -2.2726588547229767,
      "reward_std": 3.8129764549434184,
      "rewards/reward_func": -2.2726588547229767,
      "step": 50,
      "toxic_reward": 3.694073647260666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.475,
      "epoch": 0.014177693761814745,
      "format_reward": -3.75,
      "grad_norm": 0.46164554357528687,
      "image_reward": 0.24024454802274703,
      "kl": 0.000736865375074558,
      "learning_rate": 5e-06,
      "loss": 0.1005,
      "reward": -3.6363322257995607,
      "reward_std": 5.34180793762207,
      "rewards/reward_func": -3.6363322257995607,
      "step": 60,
      "toxic_reward": 4.516654038429261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.275,
      "epoch": 0.0165406427221172,
      "format_reward": -1.75,
      "grad_norm": 0.34076398611068726,
      "image_reward": 0.2581237778067589,
      "kl": 0.0006409274850739166,
      "learning_rate": 5e-06,
      "loss": -0.048,
      "reward": -1.306644481420517,
      "reward_std": 3.5849914638325573,
      "rewards/reward_func": -1.306644481420517,
      "step": 70,
      "toxic_reward": 4.222395324707032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.05,
      "epoch": 0.01890359168241966,
      "format_reward": -1.75,
      "grad_norm": 0.2942235469818115,
      "image_reward": 0.2728251129388809,
      "kl": 0.00077872859837953,
      "learning_rate": 5e-06,
      "loss": -0.018,
      "reward": -1.0979918956756591,
      "reward_std": 3.421245375275612,
      "rewards/reward_func": -1.0979918956756591,
      "step": 80,
      "toxic_reward": 4.375229549407959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.375,
      "epoch": 0.021266540642722116,
      "format_reward": -1.75,
      "grad_norm": 0.4680553376674652,
      "image_reward": 0.27020376589563155,
      "kl": 0.0006814575113821775,
      "learning_rate": 5e-06,
      "loss": 0.1758,
      "reward": -1.8029783844947815,
      "reward_std": 2.909199387952685,
      "rewards/reward_func": -1.8029783844947815,
      "step": 90,
      "toxic_reward": 3.5180059373378754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.975,
      "epoch": 0.023629489603024575,
      "format_reward": -1.75,
      "grad_norm": 0.5208232998847961,
      "image_reward": 0.28130289614200593,
      "kl": 0.0006654941505985334,
      "learning_rate": 5e-06,
      "loss": 0.0732,
      "reward": -1.5811177730560302,
      "reward_std": 3.0347108453512193,
      "rewards/reward_func": -1.5811177730560302,
      "step": 100,
      "toxic_reward": 3.8031033158302305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 59.575,
      "epoch": 0.02599243856332703,
      "format_reward": -3.5,
      "grad_norm": 0.5875898003578186,
      "image_reward": 0.2767374664545059,
      "kl": 0.0009529282746370882,
      "learning_rate": 5e-06,
      "loss": 0.013,
      "reward": -3.43455148935318,
      "reward_std": 5.033185955882073,
      "rewards/reward_func": -3.43455148935318,
      "step": 110,
      "toxic_reward": 3.8197044640779496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.825,
      "epoch": 0.02835538752362949,
      "format_reward": -2.5,
      "grad_norm": 0.9147374629974365,
      "image_reward": 0.298614501953125,
      "kl": 0.0007633624511072413,
      "learning_rate": 5e-06,
      "loss": 0.1077,
      "reward": -2.407980114221573,
      "reward_std": 4.146487069129944,
      "rewards/reward_func": -2.407980114221573,
      "step": 120,
      "toxic_reward": 3.8069980409410267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.575,
      "epoch": 0.030718336483931945,
      "format_reward": -1.5,
      "grad_norm": 0.6123144626617432,
      "image_reward": 0.26710906128088635,
      "kl": 0.000945484999101609,
      "learning_rate": 5e-06,
      "loss": -0.033,
      "reward": -1.4210234582424164,
      "reward_std": 2.5487833991646767,
      "rewards/reward_func": -1.4210234582424164,
      "step": 130,
      "toxic_reward": 3.9784648021062217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.325,
      "epoch": 0.0330812854442344,
      "format_reward": -1.5,
      "grad_norm": 0.35265249013900757,
      "image_reward": 0.2955657958984375,
      "kl": 0.001620796724455431,
      "learning_rate": 5e-06,
      "loss": 0.0518,
      "reward": -1.1245046585798264,
      "reward_std": 3.641619694232941,
      "rewards/reward_func": -1.1245046585798264,
      "step": 140,
      "toxic_reward": 3.7418821096420287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.325,
      "epoch": 0.03544423440453686,
      "format_reward": -1.75,
      "grad_norm": 0.6911599040031433,
      "image_reward": 0.301416015625,
      "kl": 0.0009025269537232816,
      "learning_rate": 5e-06,
      "loss": 0.3208,
      "reward": -1.707236361503601,
      "reward_std": 3.211209188401699,
      "rewards/reward_func": -1.707236361503601,
      "step": 150,
      "toxic_reward": 3.413761219382286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.6,
      "epoch": 0.03780718336483932,
      "format_reward": -2.5,
      "grad_norm": 0.6072728037834167,
      "image_reward": 0.28253965079784393,
      "kl": 0.0016979283303953708,
      "learning_rate": 5e-06,
      "loss": -0.0538,
      "reward": -1.9519330382347106,
      "reward_std": 3.5465006709098814,
      "rewards/reward_func": -1.9519330382347106,
      "step": 160,
      "toxic_reward": 4.008814732233684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.7,
      "epoch": 0.04017013232514178,
      "format_reward": -2.5,
      "grad_norm": 0.9174755811691284,
      "image_reward": 0.2571976251072354,
      "kl": 0.002261338901007548,
      "learning_rate": 5e-06,
      "loss": 0.1112,
      "reward": -2.123195892572403,
      "reward_std": 4.526358595490455,
      "rewards/reward_func": -2.123195892572403,
      "step": 170,
      "toxic_reward": 3.4624782469537525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.25,
      "epoch": 0.04253308128544423,
      "format_reward": -3.5,
      "grad_norm": 0.6067785024642944,
      "image_reward": 0.26939900666475297,
      "kl": 0.0012992891133762896,
      "learning_rate": 5e-06,
      "loss": 0.2051,
      "reward": -3.432029390335083,
      "reward_std": 5.464101791381836,
      "rewards/reward_func": -3.432029390335083,
      "step": 180,
      "toxic_reward": 3.7212570786476133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.175,
      "epoch": 0.04489603024574669,
      "format_reward": -1.5,
      "grad_norm": 0.32170766592025757,
      "image_reward": 0.2965630425347222,
      "kl": 0.00527564455405809,
      "learning_rate": 5e-06,
      "loss": -0.1016,
      "reward": -1.44393031001091,
      "reward_std": 3.0585690192878245,
      "rewards/reward_func": -1.44393031001091,
      "step": 190,
      "toxic_reward": 4.017925447887844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.775,
      "epoch": 0.04725897920604915,
      "format_reward": -2.0,
      "grad_norm": 0.5049771070480347,
      "image_reward": 0.28580220490694047,
      "kl": 0.003515976545168087,
      "learning_rate": 5e-06,
      "loss": 0.0836,
      "reward": -1.8655982911586761,
      "reward_std": 3.0476409645751117,
      "rewards/reward_func": -1.8655982911586761,
      "step": 200,
      "toxic_reward": 3.8450406193733215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.05,
      "epoch": 0.04962192816635161,
      "format_reward": -2.0,
      "grad_norm": 0.38584810495376587,
      "image_reward": 0.2693684895833333,
      "kl": 0.0034694685833528637,
      "learning_rate": 5e-06,
      "loss": 0.2267,
      "reward": -2.040862238407135,
      "reward_std": 3.0185029461979864,
      "rewards/reward_func": -2.040862238407135,
      "step": 210,
      "toxic_reward": 4.567277669906616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.325,
      "epoch": 0.05198487712665406,
      "format_reward": -2.25,
      "grad_norm": 0.7845410108566284,
      "image_reward": 0.27211100459098814,
      "kl": 0.0027843258751090614,
      "learning_rate": 5e-06,
      "loss": -0.0217,
      "reward": -1.9688808619976044,
      "reward_std": 4.326950389891863,
      "rewards/reward_func": -1.9688808619976044,
      "step": 220,
      "toxic_reward": 3.998746132850647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.475,
      "epoch": 0.05434782608695652,
      "format_reward": -1.5,
      "grad_norm": 0.28465747833251953,
      "image_reward": 0.28180135041475296,
      "kl": 0.0029356992337852715,
      "learning_rate": 5e-06,
      "loss": 0.1949,
      "reward": -1.8664621770381928,
      "reward_std": 3.3784094207920132,
      "rewards/reward_func": -1.8664621770381928,
      "step": 230,
      "toxic_reward": 3.4729531943798064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 61.9,
      "epoch": 0.05671077504725898,
      "format_reward": -3.25,
      "grad_norm": 0.42949026823043823,
      "image_reward": 0.27780679166316985,
      "kl": 0.00391890910686925,
      "learning_rate": 5e-06,
      "loss": 0.0808,
      "reward": -2.961669445037842,
      "reward_std": 4.58679872751236,
      "rewards/reward_func": -2.961669445037842,
      "step": 240,
      "toxic_reward": 3.08537415266037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.8,
      "epoch": 0.05907372400756144,
      "format_reward": -2.25,
      "grad_norm": 0.9381951093673706,
      "image_reward": 0.276055908203125,
      "kl": 0.017148628836730496,
      "learning_rate": 5e-06,
      "loss": -0.2303,
      "reward": -2.1199170768260958,
      "reward_std": 3.3072034239768984,
      "rewards/reward_func": -2.1199170768260958,
      "step": 250,
      "toxic_reward": 3.4872416734695433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.7,
      "epoch": 0.06143667296786389,
      "format_reward": -0.75,
      "grad_norm": 1.0572214126586914,
      "image_reward": 0.26918131560087205,
      "kl": 0.0040537358960136775,
      "learning_rate": 5e-06,
      "loss": -0.1228,
      "reward": -0.41667274236679075,
      "reward_std": 1.9968392252922058,
      "rewards/reward_func": -0.41667274236679075,
      "step": 260,
      "toxic_reward": 4.103001546859741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 31.325,
      "epoch": 0.06379962192816635,
      "format_reward": -1.75,
      "grad_norm": 0.33211401104927063,
      "image_reward": 0.26322936862707136,
      "kl": 0.010905979719245807,
      "learning_rate": 5e-06,
      "loss": 0.1079,
      "reward": -1.2869422495365144,
      "reward_std": 3.4389497309923174,
      "rewards/reward_func": -1.2869422495365144,
      "step": 270,
      "toxic_reward": 4.132273650169372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.725,
      "epoch": 0.0661625708884688,
      "format_reward": -1.0,
      "grad_norm": 1.1315058469772339,
      "image_reward": 0.28337690565321183,
      "kl": 0.003939477750100196,
      "learning_rate": 5e-06,
      "loss": 0.3379,
      "reward": -1.081434178352356,
      "reward_std": 1.8980566158890724,
      "rewards/reward_func": -1.081434178352356,
      "step": 280,
      "toxic_reward": 4.38276841905382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 31.3,
      "epoch": 0.06852551984877127,
      "format_reward": -1.75,
      "grad_norm": 0.35049131512641907,
      "image_reward": 0.27892303466796875,
      "kl": 0.04790264330804348,
      "learning_rate": 5e-06,
      "loss": -0.1138,
      "reward": -1.20261852145195,
      "reward_std": 3.7090243451297282,
      "rewards/reward_func": -1.20261852145195,
      "step": 290,
      "toxic_reward": 3.994084894657135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.925,
      "epoch": 0.07088846880907372,
      "format_reward": -2.0,
      "grad_norm": 0.5147161483764648,
      "image_reward": 0.290887451171875,
      "kl": 0.008263002592138946,
      "learning_rate": 5e-06,
      "loss": 0.0859,
      "reward": -1.9743857204914093,
      "reward_std": 3.558365413546562,
      "rewards/reward_func": -1.9743857204914093,
      "step": 300,
      "toxic_reward": 3.2862678617239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.4,
      "epoch": 0.07325141776937619,
      "format_reward": -0.25,
      "grad_norm": 0.4911198616027832,
      "image_reward": 0.27863413393497466,
      "kl": 0.0026858947356231512,
      "learning_rate": 5e-06,
      "loss": -0.1385,
      "reward": -0.005173623561859131,
      "reward_std": 1.2043775863945485,
      "rewards/reward_func": -0.005173623561859131,
      "step": 310,
      "toxic_reward": 4.392678713798523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.925,
      "epoch": 0.07561436672967864,
      "format_reward": -2.5,
      "grad_norm": 0.7166872024536133,
      "image_reward": 0.2722563561466005,
      "kl": 0.0117031121510081,
      "learning_rate": 5e-06,
      "loss": 0.1367,
      "reward": -2.320690667629242,
      "reward_std": 3.535179616510868,
      "rewards/reward_func": -2.320690667629242,
      "step": 320,
      "toxic_reward": 3.1776589486334057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.3,
      "epoch": 0.07797731568998109,
      "format_reward": -0.75,
      "grad_norm": 0.2700420618057251,
      "image_reward": 0.29552409052848816,
      "kl": 0.017672599526122212,
      "learning_rate": 5e-06,
      "loss": -0.1148,
      "reward": -0.291591414809227,
      "reward_std": 2.016152049601078,
      "rewards/reward_func": -0.291591414809227,
      "step": 330,
      "toxic_reward": 3.240800154209137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.525,
      "epoch": 0.08034026465028356,
      "format_reward": -1.75,
      "grad_norm": 0.4268760085105896,
      "image_reward": 0.29228312373161314,
      "kl": 0.0068239012965932485,
      "learning_rate": 5e-06,
      "loss": 0.1467,
      "reward": -1.9235587894916535,
      "reward_std": 2.7463727177120747,
      "rewards/reward_func": -1.9235587894916535,
      "step": 340,
      "toxic_reward": 3.535431480407715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 64.05,
      "epoch": 0.08270321361058601,
      "format_reward": -1.5,
      "grad_norm": 0.6083372235298157,
      "image_reward": 0.2844095855951309,
      "kl": 0.014765451126731933,
      "learning_rate": 5e-06,
      "loss": -0.0278,
      "reward": -1.293799924850464,
      "reward_std": 3.273481422662735,
      "rewards/reward_func": -1.293799924850464,
      "step": 350,
      "toxic_reward": 3.966331052780151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.65,
      "epoch": 0.08506616257088846,
      "format_reward": -1.25,
      "grad_norm": 0.72890704870224,
      "image_reward": 0.2809214279055595,
      "kl": 0.013393631461076439,
      "learning_rate": 5e-06,
      "loss": 0.0463,
      "reward": -1.2338840126991273,
      "reward_std": 2.9229114189743997,
      "rewards/reward_func": -1.2338840126991273,
      "step": 360,
      "toxic_reward": 3.495128685235977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.5,
      "epoch": 0.08742911153119093,
      "format_reward": -1.0,
      "grad_norm": 0.19754794239997864,
      "image_reward": 0.26820373386144636,
      "kl": 0.009714199486188591,
      "learning_rate": 5e-06,
      "loss": -0.0682,
      "reward": -0.5900760173797608,
      "reward_std": 2.301849504513666,
      "rewards/reward_func": -0.5900760173797608,
      "step": 370,
      "toxic_reward": 4.155627632141114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.325,
      "epoch": 0.08979206049149338,
      "format_reward": -1.0,
      "grad_norm": 0.7548431158065796,
      "image_reward": 0.2744639068841934,
      "kl": 0.02220306231174618,
      "learning_rate": 5e-06,
      "loss": 0.2135,
      "reward": -1.0681762412190436,
      "reward_std": 2.1873259781859815,
      "rewards/reward_func": -1.0681762412190436,
      "step": 380,
      "toxic_reward": 3.818178777396679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.75,
      "epoch": 0.09215500945179585,
      "format_reward": -1.5,
      "grad_norm": 1.6385910511016846,
      "image_reward": 0.28105672299861906,
      "kl": 0.010323460912331939,
      "learning_rate": 5e-06,
      "loss": 0.1059,
      "reward": -1.3754307508468628,
      "reward_std": 3.023836246691644,
      "rewards/reward_func": -1.3754307508468628,
      "step": 390,
      "toxic_reward": 4.011180245876313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.55,
      "epoch": 0.0945179584120983,
      "format_reward": -2.0,
      "grad_norm": 0.8115288615226746,
      "image_reward": 0.28032633662223816,
      "kl": 0.05955924341687933,
      "learning_rate": 5e-06,
      "loss": 0.0819,
      "reward": -1.5853845477104187,
      "reward_std": 4.042920933663845,
      "rewards/reward_func": -1.5853845477104187,
      "step": 400,
      "toxic_reward": 3.5872471928596497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.675,
      "epoch": 0.09688090737240075,
      "format_reward": -0.75,
      "grad_norm": 0.5388673543930054,
      "image_reward": 0.2691446923547321,
      "kl": 0.007679732237011194,
      "learning_rate": 5e-06,
      "loss": -0.0322,
      "reward": -0.41340800523757937,
      "reward_std": 2.081881234049797,
      "rewards/reward_func": -0.41340800523757937,
      "step": 410,
      "toxic_reward": 3.857707765367296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.125,
      "epoch": 0.09924385633270322,
      "format_reward": -1.75,
      "grad_norm": 0.2760399281978607,
      "image_reward": 0.2856099456548691,
      "kl": 0.06441240075509995,
      "learning_rate": 5e-06,
      "loss": 0.1586,
      "reward": -1.576817613840103,
      "reward_std": 3.0979749940335752,
      "rewards/reward_func": -1.576817613840103,
      "step": 420,
      "toxic_reward": 3.6233551859855653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.9,
      "epoch": 0.10160680529300567,
      "format_reward": -2.25,
      "grad_norm": 0.8334791660308838,
      "image_reward": 0.314910888671875,
      "kl": 0.13873190036974847,
      "learning_rate": 5e-06,
      "loss": 0.0671,
      "reward": -1.9813659265637398,
      "reward_std": 3.193006566166878,
      "rewards/reward_func": -1.9813659265637398,
      "step": 430,
      "toxic_reward": 3.297185143828392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.6,
      "epoch": 0.10396975425330812,
      "format_reward": -0.25,
      "grad_norm": 0.8734163045883179,
      "image_reward": 0.2913574203848839,
      "kl": 0.01563742496073246,
      "learning_rate": 5e-06,
      "loss": -0.0094,
      "reward": 0.012267284095287323,
      "reward_std": 1.1554903835058212,
      "rewards/reward_func": 0.012267284095287323,
      "step": 440,
      "toxic_reward": 2.416472536325455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.825,
      "epoch": 0.10633270321361059,
      "format_reward": -1.5,
      "grad_norm": 1.1138451099395752,
      "image_reward": 0.28591206669807434,
      "kl": 0.022654308984056116,
      "learning_rate": 5e-06,
      "loss": 0.0957,
      "reward": -1.2695215404033662,
      "reward_std": 2.8351699322462083,
      "rewards/reward_func": -1.2695215404033662,
      "step": 450,
      "toxic_reward": 3.3260442495346068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.475,
      "epoch": 0.10869565217391304,
      "format_reward": -0.25,
      "grad_norm": 0.6227550506591797,
      "image_reward": 0.2807729095220566,
      "kl": 0.007865939987823367,
      "learning_rate": 5e-06,
      "loss": 0.0179,
      "reward": 1.1717996835708617,
      "reward_std": 1.5977750271558762,
      "rewards/reward_func": 1.1717996835708617,
      "step": 460,
      "toxic_reward": 3.309050977230072
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.275,
      "epoch": 0.1110586011342155,
      "format_reward": -1.0,
      "grad_norm": 0.3605867624282837,
      "image_reward": 0.3162755310535431,
      "kl": 0.02495001317001879,
      "learning_rate": 5e-06,
      "loss": 0.0852,
      "reward": -1.0550554990768433,
      "reward_std": 2.7155043721199035,
      "rewards/reward_func": -1.0550554990768433,
      "step": 470,
      "toxic_reward": 3.4480915129184724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.0,
      "epoch": 0.11342155009451796,
      "format_reward": -0.5,
      "grad_norm": 0.3204725980758667,
      "image_reward": 0.32206115424633025,
      "kl": 0.011832635500468314,
      "learning_rate": 5e-06,
      "loss": 0.032,
      "reward": -0.29420808106660845,
      "reward_std": 1.479024769924581,
      "rewards/reward_func": -0.29420808106660845,
      "step": 480,
      "toxic_reward": 3.5640476822853087
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.2,
      "epoch": 0.11578449905482041,
      "format_reward": -1.75,
      "grad_norm": 0.6204938888549805,
      "image_reward": 0.26227518618106843,
      "kl": 0.03589183106087148,
      "learning_rate": 5e-06,
      "loss": 0.1186,
      "reward": -1.7148385405540467,
      "reward_std": 3.0656426630914213,
      "rewards/reward_func": -1.7148385405540467,
      "step": 490,
      "toxic_reward": 4.371080112457276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.925,
      "epoch": 0.11814744801512288,
      "format_reward": -1.75,
      "grad_norm": 0.7287388443946838,
      "image_reward": 0.2700037628412247,
      "kl": 0.045285335322842,
      "learning_rate": 5e-06,
      "loss": 0.0265,
      "reward": -1.4807079195976258,
      "reward_std": 3.789501038193703,
      "rewards/reward_func": -1.4807079195976258,
      "step": 500,
      "toxic_reward": 3.9034363865852355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.75,
      "epoch": 0.12051039697542533,
      "format_reward": -1.0,
      "grad_norm": 0.8750075697898865,
      "image_reward": 0.2932400173611111,
      "kl": 0.15442988513968886,
      "learning_rate": 5e-06,
      "loss": 0.0218,
      "reward": -0.7209997951984406,
      "reward_std": 1.843582271039486,
      "rewards/reward_func": -0.7209997951984406,
      "step": 510,
      "toxic_reward": 3.8813175095452204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.125,
      "epoch": 0.12287334593572778,
      "format_reward": -0.5,
      "grad_norm": 0.4498269259929657,
      "image_reward": 0.29094645082950593,
      "kl": 0.01979847764596343,
      "learning_rate": 5e-06,
      "loss": -0.0324,
      "reward": -0.40604341179132464,
      "reward_std": 1.4113173604011535,
      "rewards/reward_func": -0.40604341179132464,
      "step": 520,
      "toxic_reward": 4.314427596330643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.025,
      "epoch": 0.12523629489603025,
      "format_reward": -1.0,
      "grad_norm": 1.7480149269104004,
      "image_reward": 0.2472829192876816,
      "kl": 0.0556537595577538,
      "learning_rate": 5e-06,
      "loss": -0.1291,
      "reward": -1.0481307327747345,
      "reward_std": 2.317431343346834,
      "rewards/reward_func": -1.0481307327747345,
      "step": 530,
      "toxic_reward": 4.515345811843872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.2,
      "epoch": 0.1275992438563327,
      "format_reward": -1.25,
      "grad_norm": 0.39433184266090393,
      "image_reward": 0.2873850494623184,
      "kl": 0.039984302362427115,
      "learning_rate": 5e-06,
      "loss": 0.0301,
      "reward": -0.2359391689300537,
      "reward_std": 2.863342150300741,
      "rewards/reward_func": -0.2359391689300537,
      "step": 540,
      "toxic_reward": 3.940987694263458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.15,
      "epoch": 0.12996219281663515,
      "format_reward": -1.25,
      "grad_norm": 2.7985472679138184,
      "image_reward": 0.30071309208869934,
      "kl": 0.0283741801045835,
      "learning_rate": 5e-06,
      "loss": -0.0278,
      "reward": -0.4430400252342224,
      "reward_std": 2.761640505492687,
      "rewards/reward_func": -0.4430400252342224,
      "step": 550,
      "toxic_reward": 3.235564041137695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.525,
      "epoch": 0.1323251417769376,
      "format_reward": -1.0,
      "grad_norm": 1.208016037940979,
      "image_reward": 0.29123942106962203,
      "kl": 0.03811377864331007,
      "learning_rate": 5e-06,
      "loss": 0.2144,
      "reward": -1.2057244956493378,
      "reward_std": 2.0336616799235343,
      "rewards/reward_func": -1.2057244956493378,
      "step": 560,
      "toxic_reward": 3.977510142326355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.55,
      "epoch": 0.13468809073724008,
      "format_reward": -1.5,
      "grad_norm": 0.8842714428901672,
      "image_reward": 0.2724670395255089,
      "kl": 0.07012159014120697,
      "learning_rate": 5e-06,
      "loss": 0.1684,
      "reward": -1.302715817093849,
      "reward_std": 3.6691504657268523,
      "rewards/reward_func": -1.302715817093849,
      "step": 570,
      "toxic_reward": 3.246229815483093
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.6,
      "epoch": 0.13705103969754254,
      "format_reward": -0.75,
      "grad_norm": 0.7157159447669983,
      "image_reward": 0.299871826171875,
      "kl": 0.028781934920698405,
      "learning_rate": 5e-06,
      "loss": -0.0308,
      "reward": 0.11744136810302734,
      "reward_std": 2.1306695722043516,
      "rewards/reward_func": 0.11744136810302734,
      "step": 580,
      "toxic_reward": 3.35184041261673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.475,
      "epoch": 0.139413988657845,
      "format_reward": -0.25,
      "grad_norm": 0.4593754708766937,
      "image_reward": 0.2574858499897851,
      "kl": 0.05173348039388657,
      "learning_rate": 5e-06,
      "loss": -0.1241,
      "reward": 0.452265202999115,
      "reward_std": 1.2885668274015187,
      "rewards/reward_func": 0.452265202999115,
      "step": 590,
      "toxic_reward": 3.4634872145122952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.325,
      "epoch": 0.14177693761814744,
      "format_reward": -1.0,
      "grad_norm": 0.5869470834732056,
      "image_reward": 0.26802266389131546,
      "kl": 0.2022853755392134,
      "learning_rate": 5e-06,
      "loss": -0.1205,
      "reward": -0.9757636785507202,
      "reward_std": 2.408064843714237,
      "rewards/reward_func": -0.9757636785507202,
      "step": 600,
      "toxic_reward": 4.45868456363678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.45,
      "epoch": 0.1441398865784499,
      "format_reward": -2.75,
      "grad_norm": 1.1131778955459595,
      "image_reward": 0.26167353987693787,
      "kl": 0.16366879558190703,
      "learning_rate": 5e-06,
      "loss": -0.0012,
      "reward": -2.7253461956977842,
      "reward_std": 4.713953969441354,
      "rewards/reward_func": -2.7253461956977842,
      "step": 610,
      "toxic_reward": 3.5585821866989136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.65,
      "epoch": 0.14650283553875237,
      "format_reward": -1.0,
      "grad_norm": 1.6662554740905762,
      "image_reward": 0.2821828216314316,
      "kl": 0.1423144882544875,
      "learning_rate": 5e-06,
      "loss": -0.0443,
      "reward": -0.9905034899711609,
      "reward_std": 2.6423311533406375,
      "rewards/reward_func": -0.9905034899711609,
      "step": 620,
      "toxic_reward": 4.095821046829224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.85,
      "epoch": 0.14886578449905483,
      "format_reward": -1.0,
      "grad_norm": 18.956981658935547,
      "image_reward": 0.28932088166475295,
      "kl": 0.41657317453064024,
      "learning_rate": 5e-06,
      "loss": 0.0324,
      "reward": -0.8243820607662201,
      "reward_std": 2.0909267283976076,
      "rewards/reward_func": -0.8243820607662201,
      "step": 630,
      "toxic_reward": 3.2601676136255264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 29.875,
      "epoch": 0.15122873345935728,
      "format_reward": -0.75,
      "grad_norm": 1.4686508178710938,
      "image_reward": 0.29945882111787797,
      "kl": 0.28281182143837214,
      "learning_rate": 5e-06,
      "loss": 0.0769,
      "reward": -0.4713120386004448,
      "reward_std": 1.791446179151535,
      "rewards/reward_func": -0.4713120386004448,
      "step": 640,
      "toxic_reward": 3.3351209998130797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.725,
      "epoch": 0.15359168241965973,
      "format_reward": -0.5,
      "grad_norm": 2.9935286045074463,
      "image_reward": 0.2910970068640179,
      "kl": 1.0141649260884151,
      "learning_rate": 5e-06,
      "loss": -0.2174,
      "reward": -0.5139556050300598,
      "reward_std": 1.0858815148472787,
      "rewards/reward_func": -0.5139556050300598,
      "step": 650,
      "toxic_reward": 4.207416137059529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.2,
      "epoch": 0.15595463137996218,
      "format_reward": -0.75,
      "grad_norm": 3.4974160194396973,
      "image_reward": 0.29859237670898436,
      "kl": 0.03742524515837431,
      "learning_rate": 5e-06,
      "loss": -0.0491,
      "reward": -0.8688022553920746,
      "reward_std": 1.9190378237515688,
      "rewards/reward_func": -0.8688022553920746,
      "step": 660,
      "toxic_reward": 3.639171451330185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.625,
      "epoch": 0.15831758034026466,
      "format_reward": -0.75,
      "grad_norm": 0.6731751561164856,
      "image_reward": 0.2705291733145714,
      "kl": 0.1289379763416946,
      "learning_rate": 5e-06,
      "loss": 0.0339,
      "reward": -0.5425865709781647,
      "reward_std": 2.217602302134037,
      "rewards/reward_func": -0.5425865709781647,
      "step": 670,
      "toxic_reward": 3.7739344239234924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.9,
      "epoch": 0.16068052930056712,
      "format_reward": -1.0,
      "grad_norm": 0.6705069541931152,
      "image_reward": 0.2828119918704033,
      "kl": 0.09238320724107325,
      "learning_rate": 5e-06,
      "loss": -0.0765,
      "reward": -0.3722410202026367,
      "reward_std": 1.9134121721610426,
      "rewards/reward_func": -0.3722410202026367,
      "step": 680,
      "toxic_reward": 4.390137553215027
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.225,
      "epoch": 0.16304347826086957,
      "format_reward": -1.0,
      "grad_norm": 2.7068045139312744,
      "image_reward": 0.27732340693473817,
      "kl": 0.06089744158089161,
      "learning_rate": 5e-06,
      "loss": -0.0203,
      "reward": -0.6177265048027039,
      "reward_std": 2.210049830470234,
      "rewards/reward_func": -0.6177265048027039,
      "step": 690,
      "toxic_reward": 3.5699973523616793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.125,
      "epoch": 0.16540642722117202,
      "format_reward": -1.25,
      "grad_norm": 3.031416654586792,
      "image_reward": 0.2965891510248184,
      "kl": 0.8002684944309294,
      "learning_rate": 5e-06,
      "loss": 0.0719,
      "reward": -0.29744131565093995,
      "reward_std": 2.741807485371828,
      "rewards/reward_func": -0.29744131565093995,
      "step": 700,
      "toxic_reward": 3.4483383893966675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.975,
      "epoch": 0.16776937618147447,
      "format_reward": -0.25,
      "grad_norm": 3.4755773544311523,
      "image_reward": 0.2723083525896072,
      "kl": 0.24097473481670023,
      "learning_rate": 5e-06,
      "loss": 0.0494,
      "reward": -0.21520038843154907,
      "reward_std": 0.7798372395336628,
      "rewards/reward_func": -0.21520038843154907,
      "step": 710,
      "toxic_reward": 4.5303761720657345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.725,
      "epoch": 0.17013232514177692,
      "format_reward": -0.25,
      "grad_norm": 1.2503156661987305,
      "image_reward": 0.27466329038143156,
      "kl": 0.2257185777183622,
      "learning_rate": 5e-06,
      "loss": -0.0733,
      "reward": 0.11292819976806641,
      "reward_std": 1.212121632695198,
      "rewards/reward_func": 0.11292819976806641,
      "step": 720,
      "toxic_reward": 4.0655577898025514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.225,
      "epoch": 0.1724952741020794,
      "format_reward": -1.5,
      "grad_norm": 7.7392988204956055,
      "image_reward": 0.2492055267095566,
      "kl": 0.37416572365909817,
      "learning_rate": 5e-06,
      "loss": 0.0225,
      "reward": -1.0509216010570526,
      "reward_std": 3.409189415350556,
      "rewards/reward_func": -1.0509216010570526,
      "step": 730,
      "toxic_reward": 4.022808003425598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.525,
      "epoch": 0.17485822306238186,
      "format_reward": -0.5,
      "grad_norm": 4.889242172241211,
      "image_reward": 0.30042317807674407,
      "kl": 0.22789150793105364,
      "learning_rate": 5e-06,
      "loss": -0.0569,
      "reward": -0.2479497730731964,
      "reward_std": 1.3530383894219995,
      "rewards/reward_func": -0.2479497730731964,
      "step": 740,
      "toxic_reward": 3.774165117740631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.55,
      "epoch": 0.1772211720226843,
      "format_reward": -1.5,
      "grad_norm": 16.729528427124023,
      "image_reward": 0.273948161303997,
      "kl": 0.43975371681153774,
      "learning_rate": 5e-06,
      "loss": 0.1103,
      "reward": -1.793390053510666,
      "reward_std": 3.0602585028856994,
      "rewards/reward_func": -1.793390053510666,
      "step": 750,
      "toxic_reward": 3.111769822239876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.675,
      "epoch": 0.17958412098298676,
      "format_reward": -0.25,
      "grad_norm": 10.731781005859375,
      "image_reward": 0.26650288701057434,
      "kl": 0.6582286342978477,
      "learning_rate": 5e-06,
      "loss": 0.1081,
      "reward": 0.10775105953216553,
      "reward_std": 1.3219802690669895,
      "rewards/reward_func": 0.10775105953216553,
      "step": 760,
      "toxic_reward": 4.1322005033493046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.0,
      "epoch": 0.1819470699432892,
      "format_reward": -0.75,
      "grad_norm": 4.2282633781433105,
      "image_reward": 0.28914388120174406,
      "kl": 0.7939867446199059,
      "learning_rate": 5e-06,
      "loss": 0.0704,
      "reward": -0.24524924755096436,
      "reward_std": 2.0771213214844466,
      "rewards/reward_func": -0.24524924755096436,
      "step": 770,
      "toxic_reward": 3.9121114134788515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.975,
      "epoch": 0.1843100189035917,
      "format_reward": -0.5,
      "grad_norm": 8.486693382263184,
      "image_reward": 0.246868896484375,
      "kl": 1.14481502994895,
      "learning_rate": 5e-06,
      "loss": -0.0032,
      "reward": 0.28170942068099974,
      "reward_std": 2.0574716079980133,
      "rewards/reward_func": 0.28170942068099974,
      "step": 780,
      "toxic_reward": 3.4702104151248934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.15,
      "epoch": 0.18667296786389415,
      "format_reward": -0.75,
      "grad_norm": 27.51862907409668,
      "image_reward": 0.26758320927619933,
      "kl": 1.0921552445739509,
      "learning_rate": 5e-06,
      "loss": -0.3259,
      "reward": -0.5566600695252418,
      "reward_std": 1.7622592605650425,
      "rewards/reward_func": -0.5566600695252418,
      "step": 790,
      "toxic_reward": 3.4233752876520156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.45,
      "epoch": 0.1890359168241966,
      "format_reward": -0.5,
      "grad_norm": 4.040957927703857,
      "image_reward": 0.3153462767601013,
      "kl": 1.9678303502500056,
      "learning_rate": 5e-06,
      "loss": -0.1665,
      "reward": -0.010482311248779297,
      "reward_std": 1.1518827967345715,
      "rewards/reward_func": -0.010482311248779297,
      "step": 800,
      "toxic_reward": 3.6056110084056856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.5,
      "epoch": 0.19139886578449905,
      "format_reward": -0.25,
      "grad_norm": 12.718086242675781,
      "image_reward": 0.27923176884651185,
      "kl": 0.9990547701716423,
      "learning_rate": 5e-06,
      "loss": -0.0447,
      "reward": 0.1995850086212158,
      "reward_std": 1.246943424642086,
      "rewards/reward_func": 0.1995850086212158,
      "step": 810,
      "toxic_reward": 3.635990482568741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.525,
      "epoch": 0.1937618147448015,
      "format_reward": -1.25,
      "grad_norm": 5.244020938873291,
      "image_reward": 0.27026468962430955,
      "kl": 2.5741087660193442,
      "learning_rate": 5e-06,
      "loss": 0.0569,
      "reward": -1.3374125480651855,
      "reward_std": 2.818611039035022,
      "rewards/reward_func": -1.3374125480651855,
      "step": 820,
      "toxic_reward": 4.255197846889496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.25,
      "epoch": 0.19612476370510398,
      "format_reward": -0.25,
      "grad_norm": 1.3633440732955933,
      "image_reward": 0.29616292417049406,
      "kl": 0.48451304286718366,
      "learning_rate": 5e-06,
      "loss": 0.1053,
      "reward": -0.34738388657569885,
      "reward_std": 0.9195286151021719,
      "rewards/reward_func": -0.34738388657569885,
      "step": 830,
      "toxic_reward": 4.384462606906891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.75,
      "epoch": 0.19848771266540643,
      "format_reward": -0.75,
      "grad_norm": 6.93122673034668,
      "image_reward": 0.2948842361569405,
      "kl": 0.3984289012849331,
      "learning_rate": 5e-06,
      "loss": 0.007,
      "reward": -0.4061413824558258,
      "reward_std": 2.115474058687687,
      "rewards/reward_func": -0.4061413824558258,
      "step": 840,
      "toxic_reward": 2.784619116783142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.85,
      "epoch": 0.2008506616257089,
      "format_reward": -1.0,
      "grad_norm": 11.167367935180664,
      "image_reward": 0.2535125732421875,
      "kl": 0.7260896906256675,
      "learning_rate": 5e-06,
      "loss": -0.0252,
      "reward": -0.6900001287460327,
      "reward_std": 2.5411489391699433,
      "rewards/reward_func": -0.6900001287460327,
      "step": 850,
      "toxic_reward": 3.902221655845642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.35,
      "epoch": 0.20321361058601134,
      "format_reward": -0.25,
      "grad_norm": 12.129627227783203,
      "image_reward": 0.25641682744026184,
      "kl": 0.5523816287517548,
      "learning_rate": 5e-06,
      "loss": -0.0869,
      "reward": 0.027270352840423583,
      "reward_std": 1.1594479020684958,
      "rewards/reward_func": 0.027270352840423583,
      "step": 860,
      "toxic_reward": 4.19142780303955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.75,
      "epoch": 0.2055765595463138,
      "format_reward": -1.0,
      "grad_norm": 25.523523330688477,
      "image_reward": 0.28674203488561845,
      "kl": 1.1298049300909043,
      "learning_rate": 5e-06,
      "loss": 0.0639,
      "reward": -1.0763263344764709,
      "reward_std": 1.7480091962963342,
      "rewards/reward_func": -1.0763263344764709,
      "step": 870,
      "toxic_reward": 4.468152364095052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.575,
      "epoch": 0.20793950850661624,
      "format_reward": -1.0,
      "grad_norm": 3.8387675285339355,
      "image_reward": 0.26868184506893156,
      "kl": 0.9680751413106918,
      "learning_rate": 5e-06,
      "loss": -0.0833,
      "reward": -0.8666846975684166,
      "reward_std": 2.079224378615618,
      "rewards/reward_func": -0.8666846975684166,
      "step": 880,
      "toxic_reward": 3.481996048986912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.825,
      "epoch": 0.21030245746691872,
      "format_reward": -0.5,
      "grad_norm": 15.843626022338867,
      "image_reward": 0.2802464798092842,
      "kl": 0.49419727362692356,
      "learning_rate": 5e-06,
      "loss": 0.0241,
      "reward": 0.11158292293548584,
      "reward_std": 1.7106264479458333,
      "rewards/reward_func": 0.11158292293548584,
      "step": 890,
      "toxic_reward": 3.7324341177940368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.3,
      "epoch": 0.21266540642722118,
      "format_reward": -0.25,
      "grad_norm": 2.770407199859619,
      "image_reward": 0.27023824155330656,
      "kl": 0.2871086034923792,
      "learning_rate": 5e-06,
      "loss": 0.1861,
      "reward": -0.27072116136550906,
      "reward_std": 1.447587224841118,
      "rewards/reward_func": -0.27072116136550906,
      "step": 900,
      "toxic_reward": 3.426037532091141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.0,
      "epoch": 0.21502835538752363,
      "format_reward": -0.5,
      "grad_norm": 6.4211225509643555,
      "image_reward": 0.2804026290774345,
      "kl": 1.5080223519355058,
      "learning_rate": 5e-06,
      "loss": 0.0382,
      "reward": -0.10845602005720138,
      "reward_std": 1.7854840472340583,
      "rewards/reward_func": -0.10845602005720138,
      "step": 910,
      "toxic_reward": 3.3229601860046385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.4,
      "epoch": 0.21739130434782608,
      "format_reward": -1.0,
      "grad_norm": 1.846864938735962,
      "image_reward": 0.29064489238791996,
      "kl": 0.8340548906475306,
      "learning_rate": 5e-06,
      "loss": 0.0872,
      "reward": -1.217875736951828,
      "reward_std": 1.4547557694837452,
      "rewards/reward_func": -1.217875736951828,
      "step": 920,
      "toxic_reward": 4.098645766576131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.3,
      "epoch": 0.21975425330812853,
      "format_reward": -0.5,
      "grad_norm": 14.329817771911621,
      "image_reward": 0.28984171748161314,
      "kl": 0.3335365690290928,
      "learning_rate": 5e-06,
      "loss": 0.0341,
      "reward": -0.14692462086677552,
      "reward_std": 1.6654048651456832,
      "rewards/reward_func": -0.14692462086677552,
      "step": 930,
      "toxic_reward": 3.8828285098075868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.15,
      "epoch": 0.222117202268431,
      "format_reward": -1.0,
      "grad_norm": 13.11744499206543,
      "image_reward": 0.2768778458237648,
      "kl": 0.7420168094336986,
      "learning_rate": 5e-06,
      "loss": -0.1362,
      "reward": -0.5828769445419312,
      "reward_std": 2.509597599506378,
      "rewards/reward_func": -0.5828769445419312,
      "step": 940,
      "toxic_reward": 3.994591364264488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.9,
      "epoch": 0.22448015122873347,
      "format_reward": -1.25,
      "grad_norm": 1.4235849380493164,
      "image_reward": 0.2599512729793787,
      "kl": 0.23791442420333625,
      "learning_rate": 5e-06,
      "loss": 0.2081,
      "reward": -0.7265825271606445,
      "reward_std": 2.4457253187894823,
      "rewards/reward_func": -0.7265825271606445,
      "step": 950,
      "toxic_reward": 4.328470140695572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.05,
      "epoch": 0.22684310018903592,
      "format_reward": -1.0,
      "grad_norm": 10.51688003540039,
      "image_reward": 0.29052734225988386,
      "kl": 1.1104660354554654,
      "learning_rate": 5e-06,
      "loss": 0.1948,
      "reward": -0.3963636875152588,
      "reward_std": 2.6071507059037686,
      "rewards/reward_func": -0.3963636875152588,
      "step": 960,
      "toxic_reward": 3.5060137271881104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.075,
      "epoch": 0.22920604914933837,
      "format_reward": 0.0,
      "grad_norm": 0.5477933287620544,
      "image_reward": 0.2825276702642441,
      "kl": 0.24828157052397729,
      "learning_rate": 5e-06,
      "loss": 0.2364,
      "reward": -0.024850471317768096,
      "reward_std": 0.7480767840519548,
      "rewards/reward_func": -0.024850471317768096,
      "step": 970,
      "toxic_reward": 3.07701745480299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.925,
      "epoch": 0.23156899810964082,
      "format_reward": -1.25,
      "grad_norm": 6.296302318572998,
      "image_reward": 0.26927467518382603,
      "kl": 3.1552879590541125,
      "learning_rate": 5e-06,
      "loss": 0.0241,
      "reward": -0.648174649477005,
      "reward_std": 2.8984405621886253,
      "rewards/reward_func": -0.648174649477005,
      "step": 980,
      "toxic_reward": 3.881988432672289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.475,
      "epoch": 0.2339319470699433,
      "format_reward": -1.0,
      "grad_norm": 2.797386646270752,
      "image_reward": 0.2668904632329941,
      "kl": 1.7048991359770298,
      "learning_rate": 5e-06,
      "loss": -0.0828,
      "reward": -1.1502302587032318,
      "reward_std": 2.383236999064684,
      "rewards/reward_func": -1.1502302587032318,
      "step": 990,
      "toxic_reward": 4.231578087806701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.425,
      "epoch": 0.23629489603024575,
      "format_reward": -0.75,
      "grad_norm": 13.208063125610352,
      "image_reward": 0.2917307555675507,
      "kl": 0.7445122614502907,
      "learning_rate": 5e-06,
      "loss": -0.1073,
      "reward": -0.7605196535587311,
      "reward_std": 2.2064386613667013,
      "rewards/reward_func": -0.7605196535587311,
      "step": 1000,
      "toxic_reward": 3.5633171044290064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.875,
      "epoch": 0.2386578449905482,
      "format_reward": -1.0,
      "grad_norm": 10.358668327331543,
      "image_reward": 0.26257934868335725,
      "kl": 0.35015557184815405,
      "learning_rate": 5e-06,
      "loss": -0.0206,
      "reward": -0.38898804783821106,
      "reward_std": 2.7123206526041033,
      "rewards/reward_func": -0.38898804783821106,
      "step": 1010,
      "toxic_reward": 3.609158730506897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.95,
      "epoch": 0.24102079395085066,
      "format_reward": -1.0,
      "grad_norm": 9.602174758911133,
      "image_reward": 0.289794921875,
      "kl": 0.2867487147450447,
      "learning_rate": 5e-06,
      "loss": 0.0269,
      "reward": -0.4154239475727081,
      "reward_std": 2.4513496346771717,
      "rewards/reward_func": -0.4154239475727081,
      "step": 1020,
      "toxic_reward": 4.2405922412872314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.5,
      "epoch": 0.2433837429111531,
      "format_reward": -1.0,
      "grad_norm": 6.7750630378723145,
      "image_reward": 0.2876515701413155,
      "kl": 0.8189243379980325,
      "learning_rate": 5e-06,
      "loss": 0.0184,
      "reward": -0.9024024844169617,
      "reward_std": 2.123489296063781,
      "rewards/reward_func": -0.9024024844169617,
      "step": 1030,
      "toxic_reward": 3.870901381969452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.1,
      "epoch": 0.24574669187145556,
      "format_reward": -0.5,
      "grad_norm": 1.4051434993743896,
      "image_reward": 0.2766723616255654,
      "kl": 0.7713468134403229,
      "learning_rate": 5e-06,
      "loss": -0.1005,
      "reward": 0.42890325784683225,
      "reward_std": 1.7344073422253132,
      "rewards/reward_func": 0.42890325784683225,
      "step": 1040,
      "toxic_reward": 3.850937591658698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.6,
      "epoch": 0.24810964083175804,
      "format_reward": -0.5,
      "grad_norm": 10.04930591583252,
      "image_reward": 0.2845031708478928,
      "kl": 0.21945146545767785,
      "learning_rate": 5e-06,
      "loss": -0.0794,
      "reward": -0.29822829365730286,
      "reward_std": 2.0626097127795218,
      "rewards/reward_func": -0.29822829365730286,
      "step": 1050,
      "toxic_reward": 3.3056647762656213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.675,
      "epoch": 0.2504725897920605,
      "format_reward": -0.25,
      "grad_norm": 1.4483786821365356,
      "image_reward": 0.2949198380112648,
      "kl": 0.5147463826462626,
      "learning_rate": 5e-06,
      "loss": 0.0472,
      "reward": -0.4302744150161743,
      "reward_std": 0.9093868482857943,
      "rewards/reward_func": -0.4302744150161743,
      "step": 1060,
      "toxic_reward": 4.118269920349121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.125,
      "epoch": 0.252835538752363,
      "format_reward": -0.75,
      "grad_norm": 5.471806526184082,
      "image_reward": 0.3024444580078125,
      "kl": 0.924912228435278,
      "learning_rate": 5e-06,
      "loss": 0.0653,
      "reward": -0.9226927876472473,
      "reward_std": 1.8348794005811215,
      "rewards/reward_func": -0.9226927876472473,
      "step": 1070,
      "toxic_reward": 3.55495400428772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.225,
      "epoch": 0.2551984877126654,
      "format_reward": -0.5,
      "grad_norm": 6.291661739349365,
      "image_reward": 0.30248311161994934,
      "kl": 0.14056268222630025,
      "learning_rate": 5e-06,
      "loss": 0.1172,
      "reward": -0.07832016348838806,
      "reward_std": 1.6703550808131695,
      "rewards/reward_func": -0.07832016348838806,
      "step": 1080,
      "toxic_reward": 3.876679849624634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.575,
      "epoch": 0.2575614366729679,
      "format_reward": 0.0,
      "grad_norm": 5.747459411621094,
      "image_reward": 0.268257649242878,
      "kl": 0.20501487758010625,
      "learning_rate": 5e-06,
      "loss": 0.1961,
      "reward": 0.8156829088926315,
      "reward_std": 0.6415594108402729,
      "rewards/reward_func": 0.8156829088926315,
      "step": 1090,
      "toxic_reward": 4.041116189956665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.5,
      "epoch": 0.2599243856332703,
      "format_reward": -0.25,
      "grad_norm": 0.5391029715538025,
      "image_reward": 0.27643330842256547,
      "kl": 0.27743567544966935,
      "learning_rate": 5e-06,
      "loss": -0.0042,
      "reward": 0.06835275292396545,
      "reward_std": 1.1296793665736913,
      "rewards/reward_func": 0.06835275292396545,
      "step": 1100,
      "toxic_reward": 3.7508057713508607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.475,
      "epoch": 0.2622873345935728,
      "format_reward": -0.75,
      "grad_norm": 5.044631004333496,
      "image_reward": 0.2711191803216934,
      "kl": 0.08945430461317301,
      "learning_rate": 5e-06,
      "loss": 0.1402,
      "reward": -0.8795787930488587,
      "reward_std": 1.802781331539154,
      "rewards/reward_func": -0.8795787930488587,
      "step": 1110,
      "toxic_reward": 3.978659760951996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.0,
      "epoch": 0.2646502835538752,
      "format_reward": -1.25,
      "grad_norm": 10.223982810974121,
      "image_reward": 0.2896250396966934,
      "kl": 0.5244473532773555,
      "learning_rate": 5e-06,
      "loss": 0.1933,
      "reward": -0.48248053193092344,
      "reward_std": 2.971283960342407,
      "rewards/reward_func": -0.48248053193092344,
      "step": 1120,
      "toxic_reward": 3.2150497317314146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.8,
      "epoch": 0.2670132325141777,
      "format_reward": -0.5,
      "grad_norm": 3.6621553897857666,
      "image_reward": 0.2852656051516533,
      "kl": 0.5911644924432039,
      "learning_rate": 5e-06,
      "loss": 0.0576,
      "reward": -0.3013936847448349,
      "reward_std": 1.430125593394041,
      "rewards/reward_func": -0.3013936847448349,
      "step": 1130,
      "toxic_reward": 4.058745819330215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.95,
      "epoch": 0.26937618147448017,
      "format_reward": -0.5,
      "grad_norm": 24.121688842773438,
      "image_reward": 0.2795908600091934,
      "kl": 0.4301185546442866,
      "learning_rate": 5e-06,
      "loss": -0.0126,
      "reward": -0.10317457914352417,
      "reward_std": 1.667516409419477,
      "rewards/reward_func": -0.10317457914352417,
      "step": 1140,
      "toxic_reward": 4.072073769569397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.025,
      "epoch": 0.2717391304347826,
      "format_reward": 0.0,
      "grad_norm": 0.7166000604629517,
      "image_reward": 0.2804423004388809,
      "kl": 0.675014778599143,
      "learning_rate": 5e-06,
      "loss": -0.0049,
      "reward": 0.4254330635070801,
      "reward_std": 0.9621219031512738,
      "rewards/reward_func": 0.4254330635070801,
      "step": 1150,
      "toxic_reward": 3.471704053878784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.175,
      "epoch": 0.2741020793950851,
      "format_reward": -1.0,
      "grad_norm": 1.803680658340454,
      "image_reward": 0.31466064155101775,
      "kl": 0.344609697163105,
      "learning_rate": 5e-06,
      "loss": -0.1152,
      "reward": -0.5670508742332458,
      "reward_std": 2.301799529790878,
      "rewards/reward_func": -0.5670508742332458,
      "step": 1160,
      "toxic_reward": 3.554426383972168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 31.475,
      "epoch": 0.2764650283553875,
      "format_reward": 0.0,
      "grad_norm": 7.919179439544678,
      "image_reward": 0.26389770656824113,
      "kl": 0.8297407850623131,
      "learning_rate": 5e-06,
      "loss": -0.28,
      "reward": 0.23291709423065185,
      "reward_std": 0.47383863255381586,
      "rewards/reward_func": 0.23291709423065185,
      "step": 1170,
      "toxic_reward": 4.360145711898804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.0,
      "epoch": 0.27882797731569,
      "format_reward": -0.5,
      "grad_norm": 294.2972106933594,
      "image_reward": 0.2640360534191132,
      "kl": 0.9242212943732738,
      "learning_rate": 5e-06,
      "loss": 0.017,
      "reward": -0.04461590349674225,
      "reward_std": 1.7138214907608926,
      "rewards/reward_func": -0.04461590349674225,
      "step": 1180,
      "toxic_reward": 3.5669440746307375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 60.525,
      "epoch": 0.28119092627599246,
      "format_reward": -0.25,
      "grad_norm": 0.6788994669914246,
      "image_reward": 0.2832122802734375,
      "kl": 6.060492021404206,
      "learning_rate": 5e-06,
      "loss": 0.1039,
      "reward": 0.30282129645347594,
      "reward_std": 1.3184241026639938,
      "rewards/reward_func": 0.30282129645347594,
      "step": 1190,
      "toxic_reward": 3.858977997303009
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.5,
      "epoch": 0.2835538752362949,
      "format_reward": -1.0,
      "grad_norm": 2.821944236755371,
      "image_reward": 0.292755126953125,
      "kl": 0.2833241932094097,
      "learning_rate": 5e-06,
      "loss": -0.0765,
      "reward": -0.8336254239082337,
      "reward_std": 2.1170720741152764,
      "rewards/reward_func": -0.8336254239082337,
      "step": 1200,
      "toxic_reward": 4.131281018257141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.975,
      "epoch": 0.28591682419659736,
      "format_reward": -0.75,
      "grad_norm": 5.20048189163208,
      "image_reward": 0.3018681839108467,
      "kl": 0.26484427275136113,
      "learning_rate": 5e-06,
      "loss": 0.0037,
      "reward": -0.23466770052909852,
      "reward_std": 2.3572978913784026,
      "rewards/reward_func": -0.23466770052909852,
      "step": 1210,
      "toxic_reward": 3.701621878147125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.95,
      "epoch": 0.2882797731568998,
      "format_reward": -0.75,
      "grad_norm": 2.5671803951263428,
      "image_reward": 0.2591837555170059,
      "kl": 0.27887978348881004,
      "learning_rate": 5e-06,
      "loss": 0.1531,
      "reward": -0.5629445493221283,
      "reward_std": 2.2025086715817452,
      "rewards/reward_func": -0.5629445493221283,
      "step": 1220,
      "toxic_reward": 3.878066289424896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.975,
      "epoch": 0.29064272211720227,
      "format_reward": -0.25,
      "grad_norm": 1.3592997789382935,
      "image_reward": 0.2804290771484375,
      "kl": 0.7250507925637066,
      "learning_rate": 5e-06,
      "loss": -0.1268,
      "reward": 0.029623252153396607,
      "reward_std": 1.3399539720267057,
      "rewards/reward_func": 0.029623252153396607,
      "step": 1230,
      "toxic_reward": 3.5630233764648436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.675,
      "epoch": 0.29300567107750475,
      "format_reward": -1.25,
      "grad_norm": 6.867509365081787,
      "image_reward": 0.2880493178963661,
      "kl": 0.46422886326909063,
      "learning_rate": 5e-06,
      "loss": 0.0122,
      "reward": -1.0097105741500854,
      "reward_std": 2.696252405457199,
      "rewards/reward_func": -1.0097105741500854,
      "step": 1240,
      "toxic_reward": 4.076703870296479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.4,
      "epoch": 0.2953686200378072,
      "format_reward": -0.25,
      "grad_norm": 4.707825183868408,
      "image_reward": 0.256890869140625,
      "kl": 0.1788209406659007,
      "learning_rate": 5e-06,
      "loss": -0.0197,
      "reward": 0.38095744252204894,
      "reward_std": 1.2988073959946633,
      "rewards/reward_func": 0.38095744252204894,
      "step": 1250,
      "toxic_reward": 3.8400187373161314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.625,
      "epoch": 0.29773156899810965,
      "format_reward": -0.75,
      "grad_norm": 1.229298710823059,
      "image_reward": 0.313336181640625,
      "kl": 0.33243545759469273,
      "learning_rate": 5e-06,
      "loss": 0.1516,
      "reward": -0.5754710257053375,
      "reward_std": 1.8287720288150013,
      "rewards/reward_func": -0.5754710257053375,
      "step": 1260,
      "toxic_reward": 4.415339708328247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 60.25,
      "epoch": 0.3000945179584121,
      "format_reward": -0.5,
      "grad_norm": 0.5034794807434082,
      "image_reward": 0.27869771271944044,
      "kl": 0.38923515090718863,
      "learning_rate": 5e-06,
      "loss": -0.0327,
      "reward": -0.4456570327281952,
      "reward_std": 1.5328068390488625,
      "rewards/reward_func": -0.4456570327281952,
      "step": 1270,
      "toxic_reward": 3.8723622620105744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.425,
      "epoch": 0.30245746691871456,
      "format_reward": -1.0,
      "grad_norm": 1.2214823961257935,
      "image_reward": 0.2668467193841934,
      "kl": 1.2360946209169925,
      "learning_rate": 5e-06,
      "loss": 0.1227,
      "reward": -0.9184286594390869,
      "reward_std": 2.3616207716986537,
      "rewards/reward_func": -0.9184286594390869,
      "step": 1280,
      "toxic_reward": 4.0201707005500795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.525,
      "epoch": 0.30482041587901704,
      "format_reward": -0.5,
      "grad_norm": 0.6785597205162048,
      "image_reward": 0.27662353664636613,
      "kl": 0.6153190754354,
      "learning_rate": 5e-06,
      "loss": -0.0909,
      "reward": -0.025622844696044922,
      "reward_std": 1.7058033104985952,
      "rewards/reward_func": -0.025622844696044922,
      "step": 1290,
      "toxic_reward": 3.283605984598398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.475,
      "epoch": 0.30718336483931946,
      "format_reward": -0.5,
      "grad_norm": 1.5470991134643555,
      "image_reward": 0.28620096743106843,
      "kl": 1.3450787207111716,
      "learning_rate": 5e-06,
      "loss": -0.0773,
      "reward": 0.39500882625579836,
      "reward_std": 1.9240341871976852,
      "rewards/reward_func": 0.39500882625579836,
      "step": 1300,
      "toxic_reward": 3.8390918374061584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.6,
      "epoch": 0.30954631379962194,
      "format_reward": 0.0,
      "grad_norm": 4.827681541442871,
      "image_reward": 0.2871856689453125,
      "kl": 0.2589964304119349,
      "learning_rate": 5e-06,
      "loss": -0.0753,
      "reward": -0.08085522651672364,
      "reward_std": 0.7007970325648785,
      "rewards/reward_func": -0.08085522651672364,
      "step": 1310,
      "toxic_reward": 4.15708065032959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.175,
      "epoch": 0.31190926275992437,
      "format_reward": 0.0,
      "grad_norm": 2.559379816055298,
      "image_reward": 0.28839518427848815,
      "kl": 1.160063625872135,
      "learning_rate": 5e-06,
      "loss": -0.1169,
      "reward": 0.457793202996254,
      "reward_std": 0.8301180111244321,
      "rewards/reward_func": 0.457793202996254,
      "step": 1320,
      "toxic_reward": 3.847675251960754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.05,
      "epoch": 0.31427221172022685,
      "format_reward": -0.5,
      "grad_norm": 1.0227330923080444,
      "image_reward": 0.25479024201631545,
      "kl": 5.228898542746902,
      "learning_rate": 5e-06,
      "loss": 0.1453,
      "reward": -0.19808580130338668,
      "reward_std": 1.237728140875697,
      "rewards/reward_func": -0.19808580130338668,
      "step": 1330,
      "toxic_reward": 3.487361752986908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.375,
      "epoch": 0.3166351606805293,
      "format_reward": 0.0,
      "grad_norm": 2.158604383468628,
      "image_reward": 0.27274220883846284,
      "kl": 5.145803064666689,
      "learning_rate": 5e-06,
      "loss": 0.0016,
      "reward": 0.5905790150165557,
      "reward_std": 1.0763475911691784,
      "rewards/reward_func": 0.5905790150165557,
      "step": 1340,
      "toxic_reward": 3.561137008666992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.9,
      "epoch": 0.31899810964083175,
      "format_reward": -0.25,
      "grad_norm": 1.078782081604004,
      "image_reward": 0.27456156313419344,
      "kl": 4.645642199181021,
      "learning_rate": 5e-06,
      "loss": 0.0272,
      "reward": 0.0937275767326355,
      "reward_std": 1.5942428700625897,
      "rewards/reward_func": 0.0937275767326355,
      "step": 1350,
      "toxic_reward": 3.385586667060852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.05,
      "epoch": 0.32136105860113423,
      "format_reward": -0.25,
      "grad_norm": 2.4886958599090576,
      "image_reward": 0.27929331362247467,
      "kl": 0.6772738939616829,
      "learning_rate": 5e-06,
      "loss": -0.1689,
      "reward": 0.10146453976631165,
      "reward_std": 1.4149208962917328,
      "rewards/reward_func": 0.10146453976631165,
      "step": 1360,
      "toxic_reward": 4.062562417984009
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.025,
      "epoch": 0.32372400756143666,
      "format_reward": 0.0,
      "grad_norm": 0.45091304183006287,
      "image_reward": 0.26109618991613387,
      "kl": 1.1132759511470796,
      "learning_rate": 5e-06,
      "loss": 0.0773,
      "reward": 0.4344749391078949,
      "reward_std": 0.6906750492751599,
      "rewards/reward_func": 0.4344749391078949,
      "step": 1370,
      "toxic_reward": 3.89659765958786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.1,
      "epoch": 0.32608695652173914,
      "format_reward": -1.0,
      "grad_norm": 2.2919623851776123,
      "image_reward": 0.2507191985845566,
      "kl": 2.863751105964184,
      "learning_rate": 5e-06,
      "loss": 0.0426,
      "reward": -0.3381307005882263,
      "reward_std": 1.9777413787320257,
      "rewards/reward_func": -0.3381307005882263,
      "step": 1380,
      "toxic_reward": 4.168315529823303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 59.7,
      "epoch": 0.3284499054820416,
      "format_reward": 0.0,
      "grad_norm": 17.546894073486328,
      "image_reward": 0.2879852294921875,
      "kl": 1.016882681287825,
      "learning_rate": 5e-06,
      "loss": 0.065,
      "reward": -0.029438415169715883,
      "reward_std": 0.3044209867715836,
      "rewards/reward_func": -0.029438415169715883,
      "step": 1390,
      "toxic_reward": 3.8181951224803923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.425,
      "epoch": 0.33081285444234404,
      "format_reward": -0.75,
      "grad_norm": 3.9508233070373535,
      "image_reward": 0.3041224151849747,
      "kl": 1.2148886673152446,
      "learning_rate": 5e-06,
      "loss": 0.0983,
      "reward": -0.08471554517745972,
      "reward_std": 2.0540446445345877,
      "rewards/reward_func": -0.08471554517745972,
      "step": 1400,
      "toxic_reward": 4.20858781337738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.5,
      "epoch": 0.3331758034026465,
      "format_reward": -0.75,
      "grad_norm": 23.3671817779541,
      "image_reward": 0.2869578033685684,
      "kl": 9.38541857972741,
      "learning_rate": 5e-06,
      "loss": -0.0633,
      "reward": -0.21220148205757142,
      "reward_std": 2.147160884644836,
      "rewards/reward_func": -0.21220148205757142,
      "step": 1410,
      "toxic_reward": 3.646671336889267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.5,
      "epoch": 0.33553875236294894,
      "format_reward": -0.5,
      "grad_norm": 5.768739223480225,
      "image_reward": 0.29927419126033783,
      "kl": 3.124450533092022,
      "learning_rate": 5e-06,
      "loss": 0.0205,
      "reward": -0.20792179703712463,
      "reward_std": 1.7920773405581714,
      "rewards/reward_func": -0.20792179703712463,
      "step": 1420,
      "toxic_reward": 3.938745903968811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.3,
      "epoch": 0.3379017013232514,
      "format_reward": -0.75,
      "grad_norm": 7.805192947387695,
      "image_reward": 0.2781646728515625,
      "kl": 9.086061615869403,
      "learning_rate": 5e-06,
      "loss": -0.1052,
      "reward": 0.26188963651657104,
      "reward_std": 1.916423682682216,
      "rewards/reward_func": 0.26188963651657104,
      "step": 1430,
      "toxic_reward": 3.8569429397583006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.075,
      "epoch": 0.34026465028355385,
      "format_reward": 0.0,
      "grad_norm": 6.398307800292969,
      "image_reward": 0.28298187255859375,
      "kl": 3.477000297047198,
      "learning_rate": 5e-06,
      "loss": -0.1652,
      "reward": 0.2245475471019745,
      "reward_std": 0.7394228018820286,
      "rewards/reward_func": 0.2245475471019745,
      "step": 1440,
      "toxic_reward": 3.977894365787506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.375,
      "epoch": 0.34262759924385633,
      "format_reward": -0.25,
      "grad_norm": 15.553762435913086,
      "image_reward": 0.26566060483455656,
      "kl": 17.512660111114382,
      "learning_rate": 5e-06,
      "loss": -0.0446,
      "reward": 0.14810482859611512,
      "reward_std": 1.504632395505905,
      "rewards/reward_func": 0.14810482859611512,
      "step": 1450,
      "toxic_reward": 3.5254761219024657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.475,
      "epoch": 0.3449905482041588,
      "format_reward": -1.0,
      "grad_norm": 2.524869918823242,
      "image_reward": 0.2846649169921875,
      "kl": 1.9967870802618564,
      "learning_rate": 5e-06,
      "loss": -0.002,
      "reward": -1.1410660862922668,
      "reward_std": 2.0114028319716453,
      "rewards/reward_func": -1.1410660862922668,
      "step": 1460,
      "toxic_reward": 4.1038308382034305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.125,
      "epoch": 0.34735349716446123,
      "format_reward": -1.0,
      "grad_norm": 5.871716499328613,
      "image_reward": 0.2953603118658066,
      "kl": 1.2091532168909906,
      "learning_rate": 5e-06,
      "loss": 0.1278,
      "reward": -0.8882034704089165,
      "reward_std": 2.2325065452605486,
      "rewards/reward_func": -0.8882034704089165,
      "step": 1470,
      "toxic_reward": 2.88705118894577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.5,
      "epoch": 0.3497164461247637,
      "format_reward": 0.0,
      "grad_norm": 5.483914852142334,
      "image_reward": 0.2924133285880089,
      "kl": 20.523441922478376,
      "learning_rate": 5e-06,
      "loss": 0.0722,
      "reward": 0.036271828413009646,
      "reward_std": 1.0079955972731114,
      "rewards/reward_func": 0.036271828413009646,
      "step": 1480,
      "toxic_reward": 3.1371969431638718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.975,
      "epoch": 0.35207939508506614,
      "format_reward": 0.0,
      "grad_norm": 1.4849286079406738,
      "image_reward": 0.2851186111569405,
      "kl": 2.15047435965389,
      "learning_rate": 5e-06,
      "loss": 0.0946,
      "reward": 0.441963791847229,
      "reward_std": 0.4248314931988716,
      "rewards/reward_func": 0.441963791847229,
      "step": 1490,
      "toxic_reward": 3.752102476358414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.0,
      "epoch": 0.3544423440453686,
      "format_reward": -0.25,
      "grad_norm": 2.0869834423065186,
      "image_reward": 0.2911224365234375,
      "kl": 2.3197390008717775,
      "learning_rate": 5e-06,
      "loss": -0.0123,
      "reward": 0.13550712168216705,
      "reward_std": 1.141077246889472,
      "rewards/reward_func": 0.13550712168216705,
      "step": 1500,
      "toxic_reward": 3.36595538854599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.6,
      "epoch": 0.3568052930056711,
      "format_reward": -0.25,
      "grad_norm": 2.7763924598693848,
      "image_reward": 0.28095703125,
      "kl": 0.8447903416119515,
      "learning_rate": 5e-06,
      "loss": 0.1001,
      "reward": 0.07771911025047303,
      "reward_std": 1.3111265070736409,
      "rewards/reward_func": 0.07771911025047303,
      "step": 1510,
      "toxic_reward": 4.060403060913086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.575,
      "epoch": 0.3591682419659735,
      "format_reward": -0.25,
      "grad_norm": 11.143818855285645,
      "image_reward": 0.2903269439935684,
      "kl": 1.106547536328435,
      "learning_rate": 5e-06,
      "loss": -0.0008,
      "reward": 0.7432255536317826,
      "reward_std": 1.0503722863271832,
      "rewards/reward_func": 0.7432255536317826,
      "step": 1520,
      "toxic_reward": 3.4027091443538664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.825,
      "epoch": 0.361531190926276,
      "format_reward": -0.25,
      "grad_norm": 6.157534599304199,
      "image_reward": 0.2823811858892441,
      "kl": 0.7211934769526124,
      "learning_rate": 5e-06,
      "loss": -0.0288,
      "reward": 0.11932253241539001,
      "reward_std": 1.307121137715876,
      "rewards/reward_func": 0.11932253241539001,
      "step": 1530,
      "toxic_reward": 3.8783162236213684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.075,
      "epoch": 0.3638941398865784,
      "format_reward": -1.0,
      "grad_norm": 2.383302688598633,
      "image_reward": 0.28258056491613387,
      "kl": 3.830422883108258,
      "learning_rate": 5e-06,
      "loss": -0.0149,
      "reward": -0.39380887150764465,
      "reward_std": 2.854560297727585,
      "rewards/reward_func": -0.39380887150764465,
      "step": 1540,
      "toxic_reward": 3.2431194216012953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.55,
      "epoch": 0.3662570888468809,
      "format_reward": -0.25,
      "grad_norm": 2.1643450260162354,
      "image_reward": 0.2872863754630089,
      "kl": 0.3903345447033644,
      "learning_rate": 5e-06,
      "loss": 0.2399,
      "reward": 0.23153584003448485,
      "reward_std": 1.3368525609374047,
      "rewards/reward_func": 0.23153584003448485,
      "step": 1550,
      "toxic_reward": 3.452616012096405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.625,
      "epoch": 0.3686200378071834,
      "format_reward": -0.25,
      "grad_norm": 0.922444224357605,
      "image_reward": 0.29551798701286314,
      "kl": 0.9415501815266907,
      "learning_rate": 5e-06,
      "loss": 0.0285,
      "reward": 0.2152680218219757,
      "reward_std": 1.0939797786995769,
      "rewards/reward_func": 0.2152680218219757,
      "step": 1560,
      "toxic_reward": 4.278083491325378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.65,
      "epoch": 0.3709829867674858,
      "format_reward": -1.0,
      "grad_norm": 1.9485223293304443,
      "image_reward": 0.289756266772747,
      "kl": 0.52877401644364,
      "learning_rate": 5e-06,
      "loss": -0.0562,
      "reward": -0.7691292554140091,
      "reward_std": 2.194984516873956,
      "rewards/reward_func": -0.7691292554140091,
      "step": 1570,
      "toxic_reward": 3.7907654672861097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.35,
      "epoch": 0.3733459357277883,
      "format_reward": -0.75,
      "grad_norm": 4.795892238616943,
      "image_reward": 0.29136555939912795,
      "kl": 1.7273975620046258,
      "learning_rate": 5e-06,
      "loss": 0.0006,
      "reward": -0.2548545479774475,
      "reward_std": 2.3145264372229577,
      "rewards/reward_func": -0.2548545479774475,
      "step": 1580,
      "toxic_reward": 3.206251806020737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.15,
      "epoch": 0.3757088846880907,
      "format_reward": -0.25,
      "grad_norm": 1.6828984022140503,
      "image_reward": 0.29258016049861907,
      "kl": 0.27110366327688096,
      "learning_rate": 5e-06,
      "loss": 0.0295,
      "reward": 0.2889214813709259,
      "reward_std": 1.4156969770789147,
      "rewards/reward_func": 0.2889214813709259,
      "step": 1590,
      "toxic_reward": 3.8408302307128905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.075,
      "epoch": 0.3780718336483932,
      "format_reward": -0.5,
      "grad_norm": 2.8415489196777344,
      "image_reward": 0.2738067626953125,
      "kl": 1.5718746781349182,
      "learning_rate": 5e-06,
      "loss": 0.0332,
      "reward": -0.4795783460140228,
      "reward_std": 1.1532321106642485,
      "rewards/reward_func": -0.4795783460140228,
      "step": 1600,
      "toxic_reward": 3.8701359391212464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.3,
      "epoch": 0.3804347826086957,
      "format_reward": -0.5,
      "grad_norm": 0.4898248612880707,
      "image_reward": 0.28651835173368456,
      "kl": 0.5627498641610146,
      "learning_rate": 5e-06,
      "loss": 0.1267,
      "reward": -0.35464051365852356,
      "reward_std": 1.5732567172497511,
      "rewards/reward_func": -0.35464051365852356,
      "step": 1610,
      "toxic_reward": 4.116016793251037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.0,
      "epoch": 0.3827977315689981,
      "format_reward": -0.5,
      "grad_norm": 1.5352033376693726,
      "image_reward": 0.29410196989774706,
      "kl": 1.2344657305628062,
      "learning_rate": 5e-06,
      "loss": 0.1575,
      "reward": -0.4094507694244385,
      "reward_std": 1.245941134635359,
      "rewards/reward_func": -0.4094507694244385,
      "step": 1620,
      "toxic_reward": 4.569849014282227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.625,
      "epoch": 0.3851606805293006,
      "format_reward": -1.0,
      "grad_norm": 0.5829593539237976,
      "image_reward": 0.280389404296875,
      "kl": 1.36093844124116,
      "learning_rate": 5e-06,
      "loss": 0.1245,
      "reward": -0.31835838556289675,
      "reward_std": 2.00613936111331,
      "rewards/reward_func": -0.31835838556289675,
      "step": 1630,
      "toxic_reward": 4.125273871421814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.425,
      "epoch": 0.387523629489603,
      "format_reward": -0.75,
      "grad_norm": 0.8723268508911133,
      "image_reward": 0.27287851870059965,
      "kl": 0.15645003337413071,
      "learning_rate": 5e-06,
      "loss": -0.039,
      "reward": -0.8851189732551574,
      "reward_std": 2.1296220384538174,
      "rewards/reward_func": -0.8851189732551574,
      "step": 1640,
      "toxic_reward": 3.323053848743439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.35,
      "epoch": 0.3898865784499055,
      "format_reward": -0.75,
      "grad_norm": 0.14725980162620544,
      "image_reward": 0.28720601350069047,
      "kl": 1.1328919077292086,
      "learning_rate": 5e-06,
      "loss": 0.0133,
      "reward": -0.12160237431526184,
      "reward_std": 1.725741315446794,
      "rewards/reward_func": -0.12160237431526184,
      "step": 1650,
      "toxic_reward": 3.924569344520569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.05,
      "epoch": 0.39224952741020797,
      "format_reward": -0.5,
      "grad_norm": 2.200639009475708,
      "image_reward": 0.2846842437982559,
      "kl": 0.11551734725944698,
      "learning_rate": 5e-06,
      "loss": -0.0781,
      "reward": 0.11074192523956299,
      "reward_std": 1.8953823536634444,
      "rewards/reward_func": 0.11074192523956299,
      "step": 1660,
      "toxic_reward": 3.5436886310577393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 30.925,
      "epoch": 0.3946124763705104,
      "format_reward": 0.0,
      "grad_norm": 3.0496935844421387,
      "image_reward": 0.2790842682123184,
      "kl": 2.538264278974384,
      "learning_rate": 5e-06,
      "loss": -0.1096,
      "reward": 0.14284086227416992,
      "reward_std": 0.8084073163568973,
      "rewards/reward_func": 0.14284086227416992,
      "step": 1670,
      "toxic_reward": 4.144779133796692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.675,
      "epoch": 0.39697542533081287,
      "format_reward": -0.5,
      "grad_norm": 0.9690385460853577,
      "image_reward": 0.2903676345944405,
      "kl": 3.7070351759903133,
      "learning_rate": 5e-06,
      "loss": 0.1427,
      "reward": 0.008394747972488403,
      "reward_std": 1.8407307181507349,
      "rewards/reward_func": 0.008394747972488403,
      "step": 1680,
      "toxic_reward": 3.498854029178619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.875,
      "epoch": 0.3993383742911153,
      "format_reward": -0.5,
      "grad_norm": 0.6957125067710876,
      "image_reward": 0.2657012939453125,
      "kl": 0.42172617875039575,
      "learning_rate": 5e-06,
      "loss": 0.1448,
      "reward": -0.40106786489486695,
      "reward_std": 1.718069277703762,
      "rewards/reward_func": -0.40106786489486695,
      "step": 1690,
      "toxic_reward": 3.609626793861389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.375,
      "epoch": 0.4017013232514178,
      "format_reward": -0.5,
      "grad_norm": 2.07503342628479,
      "image_reward": 0.2696156814694405,
      "kl": 1.291714602895081,
      "learning_rate": 5e-06,
      "loss": 0.0722,
      "reward": -0.014362984895706176,
      "reward_std": 1.5762588312849402,
      "rewards/reward_func": -0.014362984895706176,
      "step": 1700,
      "toxic_reward": 4.394974184036255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.125,
      "epoch": 0.40406427221172025,
      "format_reward": 0.0,
      "grad_norm": 1.1231868267059326,
      "image_reward": 0.290789794921875,
      "kl": 0.21602323912084104,
      "learning_rate": 5e-06,
      "loss": -0.0932,
      "reward": 0.4133676677942276,
      "reward_std": 0.8327854365110398,
      "rewards/reward_func": 0.4133676677942276,
      "step": 1710,
      "toxic_reward": 3.955091452598572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.45,
      "epoch": 0.4064272211720227,
      "format_reward": -0.25,
      "grad_norm": 1.602283000946045,
      "image_reward": 0.2754241943359375,
      "kl": 2.6595573978964238,
      "learning_rate": 5e-06,
      "loss": -0.1005,
      "reward": 0.07846117615699769,
      "reward_std": 1.170348797738552,
      "rewards/reward_func": 0.07846117615699769,
      "step": 1720,
      "toxic_reward": 4.142733359336853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 65.9,
      "epoch": 0.40879017013232516,
      "format_reward": -1.0,
      "grad_norm": 0.5282357335090637,
      "image_reward": 0.26338195651769636,
      "kl": 0.2848859841004014,
      "learning_rate": 5e-06,
      "loss": -0.0035,
      "reward": -0.5072973608970642,
      "reward_std": 2.7491880640387536,
      "rewards/reward_func": -0.5072973608970642,
      "step": 1730,
      "toxic_reward": 4.047195649147033
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.375,
      "epoch": 0.4111531190926276,
      "format_reward": -0.25,
      "grad_norm": 1.5527747869491577,
      "image_reward": 0.2691065490245819,
      "kl": 1.2007373101077974,
      "learning_rate": 5e-06,
      "loss": -0.0239,
      "reward": -0.045976501703262326,
      "reward_std": 0.8193172802217304,
      "rewards/reward_func": -0.045976501703262326,
      "step": 1740,
      "toxic_reward": 3.446149069070816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.725,
      "epoch": 0.41351606805293006,
      "format_reward": -0.25,
      "grad_norm": 0.5118568539619446,
      "image_reward": 0.27915140688419343,
      "kl": 0.9548864349722862,
      "learning_rate": 5e-06,
      "loss": 0.1013,
      "reward": -0.10445084571838378,
      "reward_std": 0.730734084546566,
      "rewards/reward_func": -0.10445084571838378,
      "step": 1750,
      "toxic_reward": 4.5370954990386965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.25,
      "epoch": 0.4158790170132325,
      "format_reward": -0.25,
      "grad_norm": 1.8082605600357056,
      "image_reward": 0.264396159350872,
      "kl": 1.575367003493011,
      "learning_rate": 5e-06,
      "loss": 0.0632,
      "reward": 0.14499086737632752,
      "reward_std": 0.663521677441895,
      "rewards/reward_func": 0.14499086737632752,
      "step": 1760,
      "toxic_reward": 4.827451419830322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.325,
      "epoch": 0.41824196597353497,
      "format_reward": -0.25,
      "grad_norm": 0.833739697933197,
      "image_reward": 0.28918762058019637,
      "kl": 0.6164161543361842,
      "learning_rate": 5e-06,
      "loss": -0.0846,
      "reward": -0.22242847234010696,
      "reward_std": 1.0645570412278176,
      "rewards/reward_func": -0.22242847234010696,
      "step": 1770,
      "toxic_reward": 3.958344542980194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.925,
      "epoch": 0.42060491493383745,
      "format_reward": -0.5,
      "grad_norm": 0.929023027420044,
      "image_reward": 0.2808074980974197,
      "kl": 0.8390735885128379,
      "learning_rate": 5e-06,
      "loss": -0.0834,
      "reward": -0.738262277841568,
      "reward_std": 1.677246123738587,
      "rewards/reward_func": -0.738262277841568,
      "step": 1780,
      "toxic_reward": 3.8094155311584474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.4,
      "epoch": 0.4229678638941399,
      "format_reward": -0.5,
      "grad_norm": 1.0305073261260986,
      "image_reward": 0.286659748852253,
      "kl": 0.6373991215135902,
      "learning_rate": 5e-06,
      "loss": -0.0697,
      "reward": -0.2053418666124344,
      "reward_std": 1.680133179202676,
      "rewards/reward_func": -0.2053418666124344,
      "step": 1790,
      "toxic_reward": 3.8562827944755553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.8,
      "epoch": 0.42533081285444235,
      "format_reward": 0.0,
      "grad_norm": 0.9716371297836304,
      "image_reward": 0.292718505859375,
      "kl": 0.6843567499890924,
      "learning_rate": 5e-06,
      "loss": -0.0924,
      "reward": 0.7018224939703941,
      "reward_std": 0.8987518041394651,
      "rewards/reward_func": 0.7018224939703941,
      "step": 1800,
      "toxic_reward": 3.408372712135315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.25,
      "epoch": 0.4276937618147448,
      "format_reward": 0.0,
      "grad_norm": 1.081742286682129,
      "image_reward": 0.2768310546875,
      "kl": 0.7960635300725698,
      "learning_rate": 5e-06,
      "loss": 0.0443,
      "reward": -0.25897485911846163,
      "reward_std": 0.9034805342555046,
      "rewards/reward_func": -0.25897485911846163,
      "step": 1810,
      "toxic_reward": 3.7079725742340086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.3,
      "epoch": 0.43005671077504726,
      "format_reward": -0.25,
      "grad_norm": 0.6442953944206238,
      "image_reward": 0.27892710268497467,
      "kl": 0.7656038996763528,
      "learning_rate": 5e-06,
      "loss": -0.0099,
      "reward": -0.13414714336395264,
      "reward_std": 1.1088863730430603,
      "rewards/reward_func": -0.13414714336395264,
      "step": 1820,
      "toxic_reward": 3.735495138168335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.525,
      "epoch": 0.43241965973534974,
      "format_reward": -1.0,
      "grad_norm": 0.7406989336013794,
      "image_reward": 0.2804585784673691,
      "kl": 3.6395583665929734,
      "learning_rate": 5e-06,
      "loss": -0.1008,
      "reward": -0.8905552387237549,
      "reward_std": 2.38557695299387,
      "rewards/reward_func": -0.8905552387237549,
      "step": 1830,
      "toxic_reward": 3.60183764398098
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.375,
      "epoch": 0.43478260869565216,
      "format_reward": -0.25,
      "grad_norm": 1.5541785955429077,
      "image_reward": 0.30787862092256546,
      "kl": 1.104234455060214,
      "learning_rate": 5e-06,
      "loss": 0.0222,
      "reward": 0.09280971884727478,
      "reward_std": 1.7143970176577568,
      "rewards/reward_func": 0.09280971884727478,
      "step": 1840,
      "toxic_reward": 3.689550542831421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.0,
      "epoch": 0.43714555765595464,
      "format_reward": -0.25,
      "grad_norm": 0.8598329424858093,
      "image_reward": 0.2855051666498184,
      "kl": 0.16781285647302865,
      "learning_rate": 5e-06,
      "loss": -0.1435,
      "reward": 0.3788378477096558,
      "reward_std": 1.0338344363495708,
      "rewards/reward_func": 0.3788378477096558,
      "step": 1850,
      "toxic_reward": 4.1332162618637085
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.9,
      "epoch": 0.43950850661625707,
      "format_reward": -0.5,
      "grad_norm": 1.6019521951675415,
      "image_reward": 0.27197469025850296,
      "kl": 7.518688270077109,
      "learning_rate": 5e-06,
      "loss": -0.0371,
      "reward": 0.130861234664917,
      "reward_std": 1.7171866662800312,
      "rewards/reward_func": 0.130861234664917,
      "step": 1860,
      "toxic_reward": 4.243645071983337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.7,
      "epoch": 0.44187145557655955,
      "format_reward": -0.5,
      "grad_norm": 0.5758384466171265,
      "image_reward": 0.28136799931526185,
      "kl": 2.1443952365778385,
      "learning_rate": 5e-06,
      "loss": -0.0189,
      "reward": -0.18380895256996155,
      "reward_std": 1.6837687961757184,
      "rewards/reward_func": -0.18380895256996155,
      "step": 1870,
      "toxic_reward": 3.4331242620944975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.3,
      "epoch": 0.444234404536862,
      "format_reward": -0.75,
      "grad_norm": 1.5153789520263672,
      "image_reward": 0.28166198879480364,
      "kl": 1.9300499164499343,
      "learning_rate": 5e-06,
      "loss": 0.0564,
      "reward": -0.7839775577187538,
      "reward_std": 2.034397203475237,
      "rewards/reward_func": -0.7839775577187538,
      "step": 1880,
      "toxic_reward": 3.5422126829624174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.05,
      "epoch": 0.44659735349716445,
      "format_reward": 0.0,
      "grad_norm": 1.02174973487854,
      "image_reward": 0.30441080778837204,
      "kl": 5.820364655274898,
      "learning_rate": 5e-06,
      "loss": -0.1999,
      "reward": 0.5548859179019928,
      "reward_std": 0.8466346619650722,
      "rewards/reward_func": 0.5548859179019928,
      "step": 1890,
      "toxic_reward": 3.5053808212280275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.0,
      "epoch": 0.44896030245746693,
      "format_reward": -0.75,
      "grad_norm": 1.8126834630966187,
      "image_reward": 0.25828145295381544,
      "kl": 1.9232184071093799,
      "learning_rate": 5e-06,
      "loss": 0.0966,
      "reward": -0.5137902736663819,
      "reward_std": 2.415500694513321,
      "rewards/reward_func": -0.5137902736663819,
      "step": 1900,
      "toxic_reward": 3.4278686165809633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.7,
      "epoch": 0.45132325141776936,
      "format_reward": -0.5,
      "grad_norm": 0.6371603608131409,
      "image_reward": 0.2626200348138809,
      "kl": 6.273042661882937,
      "learning_rate": 5e-06,
      "loss": 0.0209,
      "reward": -0.10160770416259765,
      "reward_std": 1.7223791293799877,
      "rewards/reward_func": -0.10160770416259765,
      "step": 1910,
      "toxic_reward": 3.4677812099456786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.9,
      "epoch": 0.45368620037807184,
      "format_reward": 0.0,
      "grad_norm": 1.025303840637207,
      "image_reward": 0.27600199580192564,
      "kl": 2.9244240637868644,
      "learning_rate": 5e-06,
      "loss": 0.0036,
      "reward": 0.2618570938706398,
      "reward_std": 0.7942308865487575,
      "rewards/reward_func": 0.2618570938706398,
      "step": 1920,
      "toxic_reward": 3.214989905059338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.5,
      "epoch": 0.4560491493383743,
      "format_reward": 0.0,
      "grad_norm": 3.0306193828582764,
      "image_reward": 0.27111816257238386,
      "kl": 7.301137297973037,
      "learning_rate": 5e-06,
      "loss": -0.3058,
      "reward": 0.7629794716835022,
      "reward_std": 1.207332517206669,
      "rewards/reward_func": 0.7629794716835022,
      "step": 1930,
      "toxic_reward": 3.8610877275466917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.325,
      "epoch": 0.45841209829867674,
      "format_reward": -0.5,
      "grad_norm": 0.4994942843914032,
      "image_reward": 0.2564666748046875,
      "kl": 1.9746190145611764,
      "learning_rate": 5e-06,
      "loss": -0.056,
      "reward": 0.17883441746234893,
      "reward_std": 1.9227621294558048,
      "rewards/reward_func": 0.17883441746234893,
      "step": 1940,
      "toxic_reward": 3.5681721329689027
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.5,
      "epoch": 0.4607750472589792,
      "format_reward": -0.5,
      "grad_norm": 1.0730820894241333,
      "image_reward": 0.2937784805893898,
      "kl": 2.8218962060287596,
      "learning_rate": 5e-06,
      "loss": 0.0566,
      "reward": -0.1567411482334137,
      "reward_std": 1.654453044757247,
      "rewards/reward_func": -0.1567411482334137,
      "step": 1950,
      "toxic_reward": 3.6663838982582093
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.725,
      "epoch": 0.46313799621928164,
      "format_reward": 0.0,
      "grad_norm": 2.0345563888549805,
      "image_reward": 0.2648590087890625,
      "kl": 0.5958237243816257,
      "learning_rate": 5e-06,
      "loss": 0.0654,
      "reward": 0.12212587893009186,
      "reward_std": 0.6707309451885521,
      "rewards/reward_func": 0.12212587893009186,
      "step": 1960,
      "toxic_reward": 3.1909562170505525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.4,
      "epoch": 0.4655009451795841,
      "format_reward": -0.5,
      "grad_norm": 5.125189781188965,
      "image_reward": 0.28848724216222765,
      "kl": 1.6634003438055516,
      "learning_rate": 5e-06,
      "loss": 0.0863,
      "reward": -0.1009038507938385,
      "reward_std": 1.4750457480549812,
      "rewards/reward_func": -0.1009038507938385,
      "step": 1970,
      "toxic_reward": 4.304786968231201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.075,
      "epoch": 0.4678638941398866,
      "format_reward": -0.25,
      "grad_norm": 1.4688388109207153,
      "image_reward": 0.27630208283662794,
      "kl": 0.420011714566499,
      "learning_rate": 5e-06,
      "loss": -0.0726,
      "reward": -0.325018173456192,
      "reward_std": 1.0332348687574266,
      "rewards/reward_func": -0.325018173456192,
      "step": 1980,
      "toxic_reward": 3.5992671266198157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.85,
      "epoch": 0.47022684310018903,
      "format_reward": -0.25,
      "grad_norm": 11.723315238952637,
      "image_reward": 0.26587321013212206,
      "kl": 0.32123089879751204,
      "learning_rate": 5e-06,
      "loss": 0.0977,
      "reward": -0.41115415692329405,
      "reward_std": 1.5678910434246063,
      "rewards/reward_func": -0.41115415692329405,
      "step": 1990,
      "toxic_reward": 3.7649365305900573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.35,
      "epoch": 0.4725897920604915,
      "format_reward": -0.25,
      "grad_norm": 2.3079888820648193,
      "image_reward": 0.27147267758846283,
      "kl": 0.2777526224032044,
      "learning_rate": 5e-06,
      "loss": -0.0282,
      "reward": -0.2599769473075867,
      "reward_std": 0.731538234371692,
      "rewards/reward_func": -0.2599769473075867,
      "step": 2000,
      "toxic_reward": 4.658599400520325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.85,
      "epoch": 0.47495274102079393,
      "format_reward": -0.5,
      "grad_norm": 14.372509956359863,
      "image_reward": 0.2984934478998184,
      "kl": 3.4746980018913747,
      "learning_rate": 5e-06,
      "loss": 0.0433,
      "reward": -0.3160775646567345,
      "reward_std": 0.8356795504689216,
      "rewards/reward_func": -0.3160775646567345,
      "step": 2010,
      "toxic_reward": 3.6712876573204993
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.975,
      "epoch": 0.4773156899810964,
      "format_reward": 0.0,
      "grad_norm": 9.949368476867676,
      "image_reward": 0.2758158355951309,
      "kl": 2.603505723550916,
      "learning_rate": 5e-06,
      "loss": -0.1898,
      "reward": 0.5061412572860717,
      "reward_std": 0.6404913809150458,
      "rewards/reward_func": 0.5061412572860717,
      "step": 2020,
      "toxic_reward": 4.01279228925705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.05,
      "epoch": 0.47967863894139884,
      "format_reward": -0.5,
      "grad_norm": 11.427620887756348,
      "image_reward": 0.2567454010248184,
      "kl": 0.622926688939333,
      "learning_rate": 5e-06,
      "loss": 0.0783,
      "reward": 0.21228746175765992,
      "reward_std": 1.9739407232031225,
      "rewards/reward_func": 0.21228746175765992,
      "step": 2030,
      "toxic_reward": 3.7354461193084716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.45,
      "epoch": 0.4820415879017013,
      "format_reward": 0.0,
      "grad_norm": 4.316232204437256,
      "image_reward": 0.2718638092279434,
      "kl": 2.3269161872565745,
      "learning_rate": 5e-06,
      "loss": -0.1163,
      "reward": 0.737056265771389,
      "reward_std": 0.9669643521308899,
      "rewards/reward_func": 0.737056265771389,
      "step": 2040,
      "toxic_reward": 3.0878625586628914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.475,
      "epoch": 0.4844045368620038,
      "format_reward": -1.0,
      "grad_norm": 41.36595153808594,
      "image_reward": 0.26953938901424407,
      "kl": 0.7504621215164662,
      "learning_rate": 5e-06,
      "loss": -0.1493,
      "reward": -1.3220559000968932,
      "reward_std": 1.9624842151999473,
      "rewards/reward_func": -1.3220559000968932,
      "step": 2050,
      "toxic_reward": 3.74695360660553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.825,
      "epoch": 0.4867674858223062,
      "format_reward": 0.0,
      "grad_norm": 6.471742153167725,
      "image_reward": 0.2753570556640625,
      "kl": 0.07729073958471418,
      "learning_rate": 5e-06,
      "loss": -0.03,
      "reward": 1.3116377294063568,
      "reward_std": 1.4300442904233932,
      "rewards/reward_func": 1.3116377294063568,
      "step": 2060,
      "toxic_reward": 3.5985005378723143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.825,
      "epoch": 0.4891304347826087,
      "format_reward": 0.0,
      "grad_norm": 1.805216670036316,
      "image_reward": 0.306744384765625,
      "kl": 6.001958086341619,
      "learning_rate": 5e-06,
      "loss": -0.1945,
      "reward": 0.36415485143661497,
      "reward_std": 0.6190065078437328,
      "rewards/reward_func": 0.36415485143661497,
      "step": 2070,
      "toxic_reward": 4.081458044052124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.275,
      "epoch": 0.4914933837429111,
      "format_reward": 0.0,
      "grad_norm": 18.216772079467773,
      "image_reward": 0.2797536224126816,
      "kl": 0.49935312662273645,
      "learning_rate": 5e-06,
      "loss": 0.0342,
      "reward": 0.23056302070617676,
      "reward_std": 0.4776972606778145,
      "rewards/reward_func": 0.23056302070617676,
      "step": 2080,
      "toxic_reward": 4.019720596075058
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.075,
      "epoch": 0.4938563327032136,
      "format_reward": -0.75,
      "grad_norm": 13.060705184936523,
      "image_reward": 0.28729756474494933,
      "kl": 4.740964457206428,
      "learning_rate": 5e-06,
      "loss": 0.0645,
      "reward": -0.4479706704616547,
      "reward_std": 2.0641879491508006,
      "rewards/reward_func": -0.4479706704616547,
      "step": 2090,
      "toxic_reward": 2.7062815964221953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.575,
      "epoch": 0.4962192816635161,
      "format_reward": -0.25,
      "grad_norm": 14.017393112182617,
      "image_reward": 0.2847381591796875,
      "kl": 0.9378721818327904,
      "learning_rate": 5e-06,
      "loss": -0.0908,
      "reward": 0.4732812285423279,
      "reward_std": 1.2860259119421245,
      "rewards/reward_func": 0.4732812285423279,
      "step": 2100,
      "toxic_reward": 3.420735603570938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.075,
      "epoch": 0.4985822306238185,
      "format_reward": -0.75,
      "grad_norm": 6.193188667297363,
      "image_reward": 0.27182515412569047,
      "kl": 2.9611662749201058,
      "learning_rate": 5e-06,
      "loss": 0.0056,
      "reward": -0.19096837639808656,
      "reward_std": 1.8480727752670645,
      "rewards/reward_func": -0.19096837639808656,
      "step": 2110,
      "toxic_reward": 4.268127584457398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.8,
      "epoch": 0.500945179584121,
      "format_reward": -0.75,
      "grad_norm": 11.63723087310791,
      "image_reward": 0.2698944091796875,
      "kl": 1.1968733308836819,
      "learning_rate": 5e-06,
      "loss": 0.0042,
      "reward": -0.5995136559009552,
      "reward_std": 2.1293695636093615,
      "rewards/reward_func": -0.5995136559009552,
      "step": 2120,
      "toxic_reward": 3.746561822295189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.8,
      "epoch": 0.5033081285444234,
      "format_reward": -0.75,
      "grad_norm": 2.3855180740356445,
      "image_reward": 0.26025390625,
      "kl": 1.5614483684301377,
      "learning_rate": 5e-06,
      "loss": 0.2496,
      "reward": -0.6204059720039368,
      "reward_std": 1.9704039812088012,
      "rewards/reward_func": -0.6204059720039368,
      "step": 2130,
      "toxic_reward": 3.747698575258255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.75,
      "epoch": 0.505671077504726,
      "format_reward": 0.0,
      "grad_norm": 7.681392669677734,
      "image_reward": 0.27169952541589737,
      "kl": 3.525779527798295,
      "learning_rate": 5e-06,
      "loss": -0.154,
      "reward": 0.7122885227203369,
      "reward_std": 1.038828771188855,
      "rewards/reward_func": 0.7122885227203369,
      "step": 2140,
      "toxic_reward": 3.8024647355079653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.55,
      "epoch": 0.5080340264650284,
      "format_reward": -0.25,
      "grad_norm": 7.522043228149414,
      "image_reward": 0.2867136627435684,
      "kl": 2.352656077966094,
      "learning_rate": 5e-06,
      "loss": -0.0567,
      "reward": 0.3375007212162018,
      "reward_std": 1.1598852841183542,
      "rewards/reward_func": 0.3375007212162018,
      "step": 2150,
      "toxic_reward": 3.6138802111148833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.75,
      "epoch": 0.5103969754253308,
      "format_reward": 0.0,
      "grad_norm": 8.265325546264648,
      "image_reward": 0.2756062835454941,
      "kl": 6.923487820476294,
      "learning_rate": 5e-06,
      "loss": -0.1108,
      "reward": 0.7483027845621109,
      "reward_std": 0.5725362204015255,
      "rewards/reward_func": 0.7483027845621109,
      "step": 2160,
      "toxic_reward": 3.906574785709381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.525,
      "epoch": 0.5127599243856332,
      "format_reward": -0.75,
      "grad_norm": 21.7608642578125,
      "image_reward": 0.2696726471185684,
      "kl": 5.021715716272593,
      "learning_rate": 5e-06,
      "loss": -0.1133,
      "reward": -0.4512764573097229,
      "reward_std": 2.062841220572591,
      "rewards/reward_func": -0.4512764573097229,
      "step": 2170,
      "toxic_reward": 4.282562255859375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.425,
      "epoch": 0.5151228733459358,
      "format_reward": -0.25,
      "grad_norm": 2.369183301925659,
      "image_reward": 0.28711649775505066,
      "kl": 12.483240520581603,
      "learning_rate": 5e-06,
      "loss": 0.0658,
      "reward": -0.0087041437625885,
      "reward_std": 1.3220645122230053,
      "rewards/reward_func": -0.0087041437625885,
      "step": 2180,
      "toxic_reward": 3.781124639511108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.0,
      "epoch": 0.5174858223062382,
      "format_reward": -0.5,
      "grad_norm": 4.219491958618164,
      "image_reward": 0.27772623747587205,
      "kl": 2.453311304561794,
      "learning_rate": 5e-06,
      "loss": -0.0285,
      "reward": -0.30757330656051635,
      "reward_std": 1.7083245173096657,
      "rewards/reward_func": -0.30757330656051635,
      "step": 2190,
      "toxic_reward": 4.130738306045532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.85,
      "epoch": 0.5198487712665406,
      "format_reward": -0.5,
      "grad_norm": 6.190961837768555,
      "image_reward": 0.2818817153573036,
      "kl": 4.28942144587636,
      "learning_rate": 5e-06,
      "loss": -0.1115,
      "reward": 0.2441554695367813,
      "reward_std": 1.9595814019441604,
      "rewards/reward_func": 0.2441554695367813,
      "step": 2200,
      "toxic_reward": 3.141683894395828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.225,
      "epoch": 0.5222117202268431,
      "format_reward": -0.5,
      "grad_norm": 4.348143577575684,
      "image_reward": 0.29916890412569047,
      "kl": 0.34145298339426516,
      "learning_rate": 5e-06,
      "loss": 0.0071,
      "reward": -0.5653827100992203,
      "reward_std": 1.6975119888782502,
      "rewards/reward_func": -0.5653827100992203,
      "step": 2210,
      "toxic_reward": 3.599680471420288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.0,
      "epoch": 0.5245746691871456,
      "format_reward": -0.75,
      "grad_norm": 6.7439422607421875,
      "image_reward": 0.2785715714097023,
      "kl": 1.8124071411788463,
      "learning_rate": 5e-06,
      "loss": 0.0723,
      "reward": -0.6911701261997223,
      "reward_std": 1.9053923369385302,
      "rewards/reward_func": -0.6911701261997223,
      "step": 2220,
      "toxic_reward": 3.67071852684021
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.9,
      "epoch": 0.526937618147448,
      "format_reward": -0.25,
      "grad_norm": 5.702417373657227,
      "image_reward": 0.2697733551263809,
      "kl": 3.5654136715456843,
      "learning_rate": 5e-06,
      "loss": -0.0592,
      "reward": 0.31644179224967955,
      "reward_std": 1.338551426678896,
      "rewards/reward_func": 0.31644179224967955,
      "step": 2230,
      "toxic_reward": 4.082410860061645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.9,
      "epoch": 0.5293005671077504,
      "format_reward": -0.75,
      "grad_norm": 3.3108696937561035,
      "image_reward": 0.2754450500011444,
      "kl": 1.1358238738030195,
      "learning_rate": 5e-06,
      "loss": 0.0103,
      "reward": -0.19608908146619797,
      "reward_std": 1.9574983415892349,
      "rewards/reward_func": -0.19608908146619797,
      "step": 2240,
      "toxic_reward": 3.8882675245404243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.775,
      "epoch": 0.531663516068053,
      "format_reward": 0.0,
      "grad_norm": 3.8872108459472656,
      "image_reward": 0.2711354583501816,
      "kl": 0.6185108724981546,
      "learning_rate": 5e-06,
      "loss": -0.0331,
      "reward": 0.43025930523872374,
      "reward_std": 0.6924620851874351,
      "rewards/reward_func": 0.43025930523872374,
      "step": 2250,
      "toxic_reward": 3.741843378543854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.05,
      "epoch": 0.5340264650283554,
      "format_reward": -0.5,
      "grad_norm": 2.605905055999756,
      "image_reward": 0.24824727326631546,
      "kl": 3.812788811326027,
      "learning_rate": 5e-06,
      "loss": -0.062,
      "reward": -0.0177284836769104,
      "reward_std": 1.7159371480345726,
      "rewards/reward_func": -0.0177284836769104,
      "step": 2260,
      "toxic_reward": 3.8558017730712892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.225,
      "epoch": 0.5363894139886578,
      "format_reward": 0.0,
      "grad_norm": 4.317953109741211,
      "image_reward": 0.29388427734375,
      "kl": 0.9772842615842819,
      "learning_rate": 5e-06,
      "loss": -0.005,
      "reward": 0.24463090300559998,
      "reward_std": 0.8211262285709381,
      "rewards/reward_func": 0.24463090300559998,
      "step": 2270,
      "toxic_reward": 3.4330978095531464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.35,
      "epoch": 0.5387523629489603,
      "format_reward": -0.25,
      "grad_norm": 2.7746388912200928,
      "image_reward": 0.28372802734375,
      "kl": 0.6956694826483727,
      "learning_rate": 5e-06,
      "loss": 0.0806,
      "reward": 0.9492665678262711,
      "reward_std": 1.2596320446580649,
      "rewards/reward_func": 0.9492665678262711,
      "step": 2280,
      "toxic_reward": 3.6599619805812837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.65,
      "epoch": 0.5411153119092628,
      "format_reward": -0.75,
      "grad_norm": 24.271883010864258,
      "image_reward": 0.25230407863855364,
      "kl": 2.0102761931717397,
      "learning_rate": 5e-06,
      "loss": 0.099,
      "reward": -0.5960418626666069,
      "reward_std": 1.6162065342068672,
      "rewards/reward_func": -0.5960418626666069,
      "step": 2290,
      "toxic_reward": 3.32955624461174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.025,
      "epoch": 0.5434782608695652,
      "format_reward": -0.75,
      "grad_norm": 12.164813995361328,
      "image_reward": 0.27450052797794344,
      "kl": 1.0361489206552505,
      "learning_rate": 5e-06,
      "loss": 0.0215,
      "reward": -0.12894563674926757,
      "reward_std": 2.2585421696305277,
      "rewards/reward_func": -0.12894563674926757,
      "step": 2300,
      "toxic_reward": 3.8079848527908324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.925,
      "epoch": 0.5458412098298677,
      "format_reward": 0.0,
      "grad_norm": 4.370122909545898,
      "image_reward": 0.28968607634305954,
      "kl": 2.262423123046756,
      "learning_rate": 5e-06,
      "loss": -0.0122,
      "reward": 0.4122478127479553,
      "reward_std": 0.8819206684827805,
      "rewards/reward_func": 0.4122478127479553,
      "step": 2310,
      "toxic_reward": 3.7774435758590696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.625,
      "epoch": 0.5482041587901701,
      "format_reward": -0.25,
      "grad_norm": 4.913710594177246,
      "image_reward": 0.2981597900390625,
      "kl": 1.1325825482606888,
      "learning_rate": 5e-06,
      "loss": 0.0383,
      "reward": -0.302042031288147,
      "reward_std": 1.1343338422477245,
      "rewards/reward_func": -0.302042031288147,
      "step": 2320,
      "toxic_reward": 3.4699944481253624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.875,
      "epoch": 0.5505671077504726,
      "format_reward": -0.5,
      "grad_norm": 10.183396339416504,
      "image_reward": 0.2794362396001816,
      "kl": 2.359659927338362,
      "learning_rate": 5e-06,
      "loss": 0.127,
      "reward": -0.5543205380439759,
      "reward_std": 1.5390649776905776,
      "rewards/reward_func": -0.5543205380439759,
      "step": 2330,
      "toxic_reward": 4.130715823173523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.275,
      "epoch": 0.552930056710775,
      "format_reward": -0.25,
      "grad_norm": 29.773969650268555,
      "image_reward": 0.3009490996599197,
      "kl": 1.2122079662978649,
      "learning_rate": 5e-06,
      "loss": -0.0302,
      "reward": 0.49274033308029175,
      "reward_std": 1.2792111776769162,
      "rewards/reward_func": 0.49274033308029175,
      "step": 2340,
      "toxic_reward": 4.144988393783569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.925,
      "epoch": 0.5552930056710775,
      "format_reward": -0.25,
      "grad_norm": 1.4507733583450317,
      "image_reward": 0.27436625212430954,
      "kl": 10.124456256255508,
      "learning_rate": 5e-06,
      "loss": -0.0586,
      "reward": 0.16714471578598022,
      "reward_std": 1.1183603500947357,
      "rewards/reward_func": 0.16714471578598022,
      "step": 2350,
      "toxic_reward": 3.7719646602869035
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.45,
      "epoch": 0.55765595463138,
      "format_reward": -0.25,
      "grad_norm": 3.8344922065734863,
      "image_reward": 0.27209879606962206,
      "kl": 0.4884789928793907,
      "learning_rate": 5e-06,
      "loss": 0.0246,
      "reward": 0.7492954432964325,
      "reward_std": 1.5298523031175137,
      "rewards/reward_func": 0.7492954432964325,
      "step": 2360,
      "toxic_reward": 3.582643675804138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.925,
      "epoch": 0.5600189035916824,
      "format_reward": -0.25,
      "grad_norm": 28.500579833984375,
      "image_reward": 0.256439208984375,
      "kl": 7.240471968054772,
      "learning_rate": 5e-06,
      "loss": 0.0033,
      "reward": 0.39032529294490814,
      "reward_std": 1.3387351400218903,
      "rewards/reward_func": 0.39032529294490814,
      "step": 2370,
      "toxic_reward": 3.7680604696273803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.35,
      "epoch": 0.5623818525519849,
      "format_reward": -0.25,
      "grad_norm": 18.509540557861328,
      "image_reward": 0.25091654509305955,
      "kl": 2.3443214535713195,
      "learning_rate": 5e-06,
      "loss": -0.0852,
      "reward": -0.013416659832000733,
      "reward_std": 1.2783805396407844,
      "rewards/reward_func": -0.013416659832000733,
      "step": 2380,
      "toxic_reward": 3.937808632850647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.125,
      "epoch": 0.5647448015122873,
      "format_reward": -0.5,
      "grad_norm": 11.650871276855469,
      "image_reward": 0.29215189516544343,
      "kl": 0.3515282288193703,
      "learning_rate": 5e-06,
      "loss": -0.0259,
      "reward": 0.1742587387561798,
      "reward_std": 1.8562648460268973,
      "rewards/reward_func": 0.1742587387561798,
      "step": 2390,
      "toxic_reward": 3.7724621415138246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.55,
      "epoch": 0.5671077504725898,
      "format_reward": -1.0,
      "grad_norm": 20.670705795288086,
      "image_reward": 0.26702982634305955,
      "kl": 2.7752922803163527,
      "learning_rate": 5e-06,
      "loss": 0.1898,
      "reward": -0.49167909026145934,
      "reward_std": 2.5721775129437447,
      "rewards/reward_func": -0.49167909026145934,
      "step": 2400,
      "toxic_reward": 3.612065541744232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.75,
      "epoch": 0.5694706994328923,
      "format_reward": -0.25,
      "grad_norm": 5.918033599853516,
      "image_reward": 0.27968953400850294,
      "kl": 1.1868829876184464,
      "learning_rate": 5e-06,
      "loss": 0.0339,
      "reward": -0.041136431694030764,
      "reward_std": 1.1883981741964817,
      "rewards/reward_func": -0.041136431694030764,
      "step": 2410,
      "toxic_reward": 4.002831280231476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.4,
      "epoch": 0.5718336483931947,
      "format_reward": -0.25,
      "grad_norm": 5.842867851257324,
      "image_reward": 0.27998046875,
      "kl": 0.9403334192931652,
      "learning_rate": 5e-06,
      "loss": 0.0547,
      "reward": 0.23068565130233765,
      "reward_std": 1.2439154148101808,
      "rewards/reward_func": 0.23068565130233765,
      "step": 2420,
      "toxic_reward": 3.8584881067276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.2,
      "epoch": 0.5741965973534972,
      "format_reward": -0.25,
      "grad_norm": 13.205660820007324,
      "image_reward": 0.2850880965590477,
      "kl": 1.6154363751411438,
      "learning_rate": 5e-06,
      "loss": -0.0775,
      "reward": 0.4115023612976074,
      "reward_std": 1.0730943327769638,
      "rewards/reward_func": 0.4115023612976074,
      "step": 2430,
      "toxic_reward": 4.400762820243836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.925,
      "epoch": 0.5765595463137996,
      "format_reward": -0.25,
      "grad_norm": 3.637028455734253,
      "image_reward": 0.26606852263212205,
      "kl": 1.6208242058753968,
      "learning_rate": 5e-06,
      "loss": 0.0364,
      "reward": -0.5815495431423188,
      "reward_std": 1.270220142416656,
      "rewards/reward_func": -0.5815495431423188,
      "step": 2440,
      "toxic_reward": 3.934324860572815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.35,
      "epoch": 0.5789224952741021,
      "format_reward": -0.75,
      "grad_norm": 11.621758460998535,
      "image_reward": 0.28047332763671873,
      "kl": 0.7798056200146675,
      "learning_rate": 5e-06,
      "loss": 0.0519,
      "reward": 0.1087100327014923,
      "reward_std": 2.0828719630837442,
      "rewards/reward_func": 0.1087100327014923,
      "step": 2450,
      "toxic_reward": 2.8291834026575087
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.95,
      "epoch": 0.5812854442344045,
      "format_reward": 0.0,
      "grad_norm": 9.702945709228516,
      "image_reward": 0.28443044126033784,
      "kl": 1.73483949303627,
      "learning_rate": 5e-06,
      "loss": 0.0958,
      "reward": 0.28035863041877745,
      "reward_std": 0.5182013310492039,
      "rewards/reward_func": 0.28035863041877745,
      "step": 2460,
      "toxic_reward": 3.8520292162895204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.925,
      "epoch": 0.583648393194707,
      "format_reward": -0.5,
      "grad_norm": 18.073659896850586,
      "image_reward": 0.24947459101676941,
      "kl": 2.9204909898340703,
      "learning_rate": 5e-06,
      "loss": -0.1939,
      "reward": 0.42990538477897644,
      "reward_std": 1.8428901416249572,
      "rewards/reward_func": 0.42990538477897644,
      "step": 2470,
      "toxic_reward": 3.7781980872154235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.175,
      "epoch": 0.5860113421550095,
      "format_reward": -0.5,
      "grad_norm": 4.270178318023682,
      "image_reward": 0.28282063752412795,
      "kl": 0.48990702964365485,
      "learning_rate": 5e-06,
      "loss": 0.0624,
      "reward": -0.30424859523773196,
      "reward_std": 1.5560518722981214,
      "rewards/reward_func": -0.30424859523773196,
      "step": 2480,
      "toxic_reward": 4.44784414768219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.9,
      "epoch": 0.5883742911153119,
      "format_reward": -0.5,
      "grad_norm": 7.575175762176514,
      "image_reward": 0.2695292145013809,
      "kl": 1.2654437847435474,
      "learning_rate": 5e-06,
      "loss": -0.1158,
      "reward": -0.44633115231990816,
      "reward_std": 1.8826897315680982,
      "rewards/reward_func": -0.44633115231990816,
      "step": 2490,
      "toxic_reward": 3.8135931372642515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.3,
      "epoch": 0.5907372400756143,
      "format_reward": 0.0,
      "grad_norm": 24.015722274780273,
      "image_reward": 0.26897684782743453,
      "kl": 5.640305678918958,
      "learning_rate": 5e-06,
      "loss": -0.1054,
      "reward": 0.6214121818542481,
      "reward_std": 0.9682584583759308,
      "rewards/reward_func": 0.6214121818542481,
      "step": 2500,
      "toxic_reward": 3.9037705421447755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.125,
      "epoch": 0.5931001890359168,
      "format_reward": -0.25,
      "grad_norm": 13.069973945617676,
      "image_reward": 0.2854502350091934,
      "kl": 11.71274044290185,
      "learning_rate": 5e-06,
      "loss": -0.077,
      "reward": -0.3511055693030357,
      "reward_std": 1.0736159782391042,
      "rewards/reward_func": -0.3511055693030357,
      "step": 2510,
      "toxic_reward": 3.7281174302101134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.375,
      "epoch": 0.5954631379962193,
      "format_reward": 0.0,
      "grad_norm": 2.0403361320495605,
      "image_reward": 0.2833099365234375,
      "kl": 0.6411756843328476,
      "learning_rate": 5e-06,
      "loss": 0.0435,
      "reward": 0.5592477023601532,
      "reward_std": 0.8428021136671304,
      "rewards/reward_func": 0.5592477023601532,
      "step": 2520,
      "toxic_reward": 3.6056689500808714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.925,
      "epoch": 0.5978260869565217,
      "format_reward": -0.5,
      "grad_norm": 2.7234652042388916,
      "image_reward": 0.2631998687982559,
      "kl": 2.588300554268062,
      "learning_rate": 5e-06,
      "loss": -0.0954,
      "reward": -0.11296717822551727,
      "reward_std": 1.059992153197527,
      "rewards/reward_func": -0.11296717822551727,
      "step": 2530,
      "toxic_reward": 4.310960650444031
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.8,
      "epoch": 0.6001890359168242,
      "format_reward": 0.0,
      "grad_norm": 1.746839165687561,
      "image_reward": 0.27794291228055956,
      "kl": 0.12578147873282433,
      "learning_rate": 5e-06,
      "loss": -0.0894,
      "reward": 0.6603235125541687,
      "reward_std": 0.5662866534665227,
      "rewards/reward_func": 0.6603235125541687,
      "step": 2540,
      "toxic_reward": 4.165549850463867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 65.2,
      "epoch": 0.6025519848771267,
      "format_reward": 0.0,
      "grad_norm": 1.1635066270828247,
      "image_reward": 0.2584126806921429,
      "kl": 16.10209010541439,
      "learning_rate": 5e-06,
      "loss": -0.0047,
      "reward": 0.9701344430446625,
      "reward_std": 0.8910946477204561,
      "rewards/reward_func": 0.9701344430446625,
      "step": 2550,
      "toxic_reward": 3.8731188111835055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.975,
      "epoch": 0.6049149338374291,
      "format_reward": -1.0,
      "grad_norm": 3.505110502243042,
      "image_reward": 0.2755279541015625,
      "kl": 1.5500462669879198,
      "learning_rate": 5e-06,
      "loss": 0.0015,
      "reward": -0.1658882439136505,
      "reward_std": 2.0384394701570274,
      "rewards/reward_func": -0.1658882439136505,
      "step": 2560,
      "toxic_reward": 3.8778061270713806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 30.8,
      "epoch": 0.6072778827977315,
      "format_reward": 0.0,
      "grad_norm": 8.120704650878906,
      "image_reward": 0.2892588287591934,
      "kl": 2.1680047139525414,
      "learning_rate": 5e-06,
      "loss": 0.0399,
      "reward": 0.6697697341442108,
      "reward_std": 1.024929089844227,
      "rewards/reward_func": 0.6697697341442108,
      "step": 2570,
      "toxic_reward": 3.547108954191208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 63.225,
      "epoch": 0.6096408317580341,
      "format_reward": -1.0,
      "grad_norm": 9.57001781463623,
      "image_reward": 0.2734588623046875,
      "kl": 0.8948870234191417,
      "learning_rate": 5e-06,
      "loss": 0.0951,
      "reward": -0.7226251482963562,
      "reward_std": 2.448101815581322,
      "rewards/reward_func": -0.7226251482963562,
      "step": 2580,
      "toxic_reward": 4.320083689689636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.1,
      "epoch": 0.6120037807183365,
      "format_reward": 0.0,
      "grad_norm": 2.0496883392333984,
      "image_reward": 0.2865132659673691,
      "kl": 2.8105035655200483,
      "learning_rate": 5e-06,
      "loss": -0.0797,
      "reward": 0.568730728328228,
      "reward_std": 0.6556393213570118,
      "rewards/reward_func": 0.568730728328228,
      "step": 2590,
      "toxic_reward": 3.725440341234207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.075,
      "epoch": 0.6143667296786389,
      "format_reward": 0.0,
      "grad_norm": 10.353742599487305,
      "image_reward": 0.2710174560546875,
      "kl": 0.6778285041451454,
      "learning_rate": 5e-06,
      "loss": 0.0379,
      "reward": 0.2569525420665741,
      "reward_std": 0.597846270352602,
      "rewards/reward_func": 0.2569525420665741,
      "step": 2600,
      "toxic_reward": 4.306404328346252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.2,
      "epoch": 0.6167296786389413,
      "format_reward": -0.5,
      "grad_norm": 3.9594945907592773,
      "image_reward": 0.28432718813419344,
      "kl": 0.5540166199207306,
      "learning_rate": 5e-06,
      "loss": 0.0047,
      "reward": 0.5912085831165313,
      "reward_std": 1.5809811264276505,
      "rewards/reward_func": 0.5912085831165313,
      "step": 2610,
      "toxic_reward": 4.350194215774536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.0,
      "epoch": 0.6190926275992439,
      "format_reward": -0.25,
      "grad_norm": 7.203413963317871,
      "image_reward": 0.26516723483800886,
      "kl": 1.199559571594,
      "learning_rate": 5e-06,
      "loss": -0.0012,
      "reward": 0.2267006203532219,
      "reward_std": 1.142584490031004,
      "rewards/reward_func": 0.2267006203532219,
      "step": 2620,
      "toxic_reward": 3.9258982062339784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.95,
      "epoch": 0.6214555765595463,
      "format_reward": -0.25,
      "grad_norm": 6.998039722442627,
      "image_reward": 0.2901885986328125,
      "kl": 27.16859985589981,
      "learning_rate": 5e-06,
      "loss": -0.026,
      "reward": 0.13268216848373413,
      "reward_std": 1.2143183693289756,
      "rewards/reward_func": 0.13268216848373413,
      "step": 2630,
      "toxic_reward": 3.556568074226379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.8,
      "epoch": 0.6238185255198487,
      "format_reward": -0.25,
      "grad_norm": 13.862479209899902,
      "image_reward": 0.27274271547794343,
      "kl": 2.363949555903673,
      "learning_rate": 5e-06,
      "loss": -0.1093,
      "reward": 0.21172123551368713,
      "reward_std": 1.788407751917839,
      "rewards/reward_func": 0.21172123551368713,
      "step": 2640,
      "toxic_reward": 3.530658257007599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.45,
      "epoch": 0.6261814744801513,
      "format_reward": 0.0,
      "grad_norm": 6.451826095581055,
      "image_reward": 0.25544840544462205,
      "kl": 0.9077189475297928,
      "learning_rate": 5e-06,
      "loss": -0.1204,
      "reward": 0.17604875564575195,
      "reward_std": 0.7731596916913986,
      "rewards/reward_func": 0.17604875564575195,
      "step": 2650,
      "toxic_reward": 3.655298948287964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.375,
      "epoch": 0.6285444234404537,
      "format_reward": -0.25,
      "grad_norm": 15.447392463684082,
      "image_reward": 0.28090617060661316,
      "kl": 1.5149286333471537,
      "learning_rate": 5e-06,
      "loss": -0.0393,
      "reward": 0.6538720428943634,
      "reward_std": 1.4380803421139716,
      "rewards/reward_func": 0.6538720428943634,
      "step": 2660,
      "toxic_reward": 3.8757722854614256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.525,
      "epoch": 0.6309073724007561,
      "format_reward": -0.5,
      "grad_norm": 5.443056583404541,
      "image_reward": 0.28455810546875,
      "kl": 2.1727461591362953,
      "learning_rate": 5e-06,
      "loss": 0.0129,
      "reward": -0.5597851276397705,
      "reward_std": 1.4988839238882066,
      "rewards/reward_func": -0.5597851276397705,
      "step": 2670,
      "toxic_reward": 3.852312761545181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.575,
      "epoch": 0.6332703213610587,
      "format_reward": 0.0,
      "grad_norm": 6.4276652336120605,
      "image_reward": 0.2711863175034523,
      "kl": 2.061120516061783,
      "learning_rate": 5e-06,
      "loss": -0.1249,
      "reward": 0.4250785157084465,
      "reward_std": 0.8246009856462478,
      "rewards/reward_func": 0.4250785157084465,
      "step": 2680,
      "toxic_reward": 3.8009597778320314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.375,
      "epoch": 0.6356332703213611,
      "format_reward": 0.0,
      "grad_norm": 1.9922189712524414,
      "image_reward": 0.2824055999517441,
      "kl": 0.8832020409405231,
      "learning_rate": 5e-06,
      "loss": 0.0029,
      "reward": 0.2428468108177185,
      "reward_std": 0.7863198474049569,
      "rewards/reward_func": 0.2428468108177185,
      "step": 2690,
      "toxic_reward": 3.925771975517273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.375,
      "epoch": 0.6379962192816635,
      "format_reward": -0.25,
      "grad_norm": 3.2788710594177246,
      "image_reward": 0.2625895172357559,
      "kl": 0.3067374438047409,
      "learning_rate": 5e-06,
      "loss": 0.0585,
      "reward": 0.0673605427145958,
      "reward_std": 1.2387039607390762,
      "rewards/reward_func": 0.0673605427145958,
      "step": 2700,
      "toxic_reward": 3.223780316114426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.7,
      "epoch": 0.6403591682419659,
      "format_reward": -0.5,
      "grad_norm": 18.068998336791992,
      "image_reward": 0.28381652683019637,
      "kl": 4.5763449721038345,
      "learning_rate": 5e-06,
      "loss": 0.0303,
      "reward": -0.08654462695121765,
      "reward_std": 1.6389019638299942,
      "rewards/reward_func": -0.08654462695121765,
      "step": 2710,
      "toxic_reward": 3.9132798612117767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.75,
      "epoch": 0.6427221172022685,
      "format_reward": 0.0,
      "grad_norm": 16.331071853637695,
      "image_reward": 0.2924163818359375,
      "kl": 1.1359277203679086,
      "learning_rate": 5e-06,
      "loss": -0.0706,
      "reward": 0.6057616770267487,
      "reward_std": 0.7651574447751045,
      "rewards/reward_func": 0.6057616770267487,
      "step": 2720,
      "toxic_reward": 3.8298017740249635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.725,
      "epoch": 0.6450850661625709,
      "format_reward": -0.25,
      "grad_norm": 2.152521848678589,
      "image_reward": 0.3041951507329941,
      "kl": 8.785355818271636,
      "learning_rate": 5e-06,
      "loss": 0.0118,
      "reward": 0.3512896567583084,
      "reward_std": 1.3057980645447969,
      "rewards/reward_func": 0.3512896567583084,
      "step": 2730,
      "toxic_reward": 3.397814577817917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.675,
      "epoch": 0.6474480151228733,
      "format_reward": 0.0,
      "grad_norm": 20.01748275756836,
      "image_reward": 0.29010823667049407,
      "kl": 3.5924226850271226,
      "learning_rate": 5e-06,
      "loss": -0.1142,
      "reward": 0.7106038928031921,
      "reward_std": 0.8158069387078285,
      "rewards/reward_func": 0.7106038928031921,
      "step": 2740,
      "toxic_reward": 4.0692403554916385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.7,
      "epoch": 0.6498109640831758,
      "format_reward": -0.75,
      "grad_norm": 11.965126037597656,
      "image_reward": 0.26697489619255066,
      "kl": 1.9964583709836006,
      "learning_rate": 5e-06,
      "loss": -0.0406,
      "reward": -0.07322075963020325,
      "reward_std": 1.999936766922474,
      "rewards/reward_func": -0.07322075963020325,
      "step": 2750,
      "toxic_reward": 3.366811156272888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.475,
      "epoch": 0.6521739130434783,
      "format_reward": -0.5,
      "grad_norm": 26.80545997619629,
      "image_reward": 0.24265645444393158,
      "kl": 2.707187344133854,
      "learning_rate": 5e-06,
      "loss": 0.0971,
      "reward": -0.5061075001955032,
      "reward_std": 1.755505845695734,
      "rewards/reward_func": -0.5061075001955032,
      "step": 2760,
      "toxic_reward": 3.8667294502258303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.025,
      "epoch": 0.6545368620037807,
      "format_reward": 0.0,
      "grad_norm": 5.15554141998291,
      "image_reward": 0.271905517578125,
      "kl": 4.915832757204771,
      "learning_rate": 5e-06,
      "loss": -0.0126,
      "reward": 0.8192368298768997,
      "reward_std": 0.41571362912654874,
      "rewards/reward_func": 0.8192368298768997,
      "step": 2770,
      "toxic_reward": 4.089378929138183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.075,
      "epoch": 0.6568998109640832,
      "format_reward": 0.0,
      "grad_norm": 2.028783082962036,
      "image_reward": 0.27303365170955657,
      "kl": 1.8947554275393486,
      "learning_rate": 5e-06,
      "loss": 0.0468,
      "reward": -0.1263785183429718,
      "reward_std": 0.7480042926967144,
      "rewards/reward_func": -0.1263785183429718,
      "step": 2780,
      "toxic_reward": 4.165195155143738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.125,
      "epoch": 0.6592627599243857,
      "format_reward": -0.5,
      "grad_norm": 17.42000961303711,
      "image_reward": 0.2864379853010178,
      "kl": 1.6738548278808594,
      "learning_rate": 5e-06,
      "loss": -0.0375,
      "reward": -0.2114594280719757,
      "reward_std": 1.560011611506343,
      "rewards/reward_func": -0.2114594280719757,
      "step": 2790,
      "toxic_reward": 4.024951922893524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.075,
      "epoch": 0.6616257088846881,
      "format_reward": 0.0,
      "grad_norm": 5.910866737365723,
      "image_reward": 0.2764821395277977,
      "kl": 2.9146203480660917,
      "learning_rate": 5e-06,
      "loss": -0.0104,
      "reward": 0.6828193128108978,
      "reward_std": 0.7127262264490127,
      "rewards/reward_func": 0.6828193128108978,
      "step": 2800,
      "toxic_reward": 4.108976912498474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.225,
      "epoch": 0.6639886578449905,
      "format_reward": -0.25,
      "grad_norm": 13.787774085998535,
      "image_reward": 0.2695404052734375,
      "kl": 2.044136567413807,
      "learning_rate": 5e-06,
      "loss": 0.1481,
      "reward": 0.1416476845741272,
      "reward_std": 0.9124870980158448,
      "rewards/reward_func": 0.1416476845741272,
      "step": 2810,
      "toxic_reward": 3.9404671788215637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.35,
      "epoch": 0.666351606805293,
      "format_reward": -0.25,
      "grad_norm": 9.458231925964355,
      "image_reward": 0.27503865361213686,
      "kl": 12.555490608513356,
      "learning_rate": 5e-06,
      "loss": 0.015,
      "reward": -0.398735374212265,
      "reward_std": 1.3145878296345472,
      "rewards/reward_func": -0.398735374212265,
      "step": 2820,
      "toxic_reward": 3.8166601181030275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.575,
      "epoch": 0.6687145557655955,
      "format_reward": 0.0,
      "grad_norm": 5.239807605743408,
      "image_reward": 0.2913035064935684,
      "kl": 4.1338134072721004,
      "learning_rate": 5e-06,
      "loss": -0.0082,
      "reward": 0.09673230051994323,
      "reward_std": 0.5237030681222677,
      "rewards/reward_func": 0.09673230051994323,
      "step": 2830,
      "toxic_reward": 3.791787397861481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.05,
      "epoch": 0.6710775047258979,
      "format_reward": 0.0,
      "grad_norm": 3.1467976570129395,
      "image_reward": 0.2838506057858467,
      "kl": 18.177365225553512,
      "learning_rate": 5e-06,
      "loss": 0.21,
      "reward": 0.40501208901405333,
      "reward_std": 0.894443211145699,
      "rewards/reward_func": 0.40501208901405333,
      "step": 2840,
      "toxic_reward": 3.982026219367981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.05,
      "epoch": 0.6734404536862004,
      "format_reward": -0.25,
      "grad_norm": 4.421890735626221,
      "image_reward": 0.2822255462408066,
      "kl": 4.33959369957447,
      "learning_rate": 5e-06,
      "loss": -0.0624,
      "reward": -0.05263040065765381,
      "reward_std": 1.2849599719047546,
      "rewards/reward_func": -0.05263040065765381,
      "step": 2850,
      "toxic_reward": 3.633159136772156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.2,
      "epoch": 0.6758034026465028,
      "format_reward": 0.0,
      "grad_norm": 2.5038645267486572,
      "image_reward": 0.2830434158444405,
      "kl": 0.6373848512768745,
      "learning_rate": 5e-06,
      "loss": -0.0724,
      "reward": 0.627695482969284,
      "reward_std": 0.8375864863395691,
      "rewards/reward_func": 0.627695482969284,
      "step": 2860,
      "toxic_reward": 2.48615984916687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.45,
      "epoch": 0.6781663516068053,
      "format_reward": -0.75,
      "grad_norm": 13.282075881958008,
      "image_reward": 0.27708842009305956,
      "kl": 0.9827784240245819,
      "learning_rate": 5e-06,
      "loss": 0.0701,
      "reward": -0.892215234041214,
      "reward_std": 2.255379121750593,
      "rewards/reward_func": -0.892215234041214,
      "step": 2870,
      "toxic_reward": 3.7220635175704957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.525,
      "epoch": 0.6805293005671077,
      "format_reward": 0.0,
      "grad_norm": 12.856422424316406,
      "image_reward": 0.2848948180675507,
      "kl": 1.0351120814681054,
      "learning_rate": 5e-06,
      "loss": 0.0319,
      "reward": 0.2441805601119995,
      "reward_std": 0.7333651419728995,
      "rewards/reward_func": 0.2441805601119995,
      "step": 2880,
      "toxic_reward": 3.4050124049186707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.925,
      "epoch": 0.6828922495274102,
      "format_reward": 0.0,
      "grad_norm": 17.430034637451172,
      "image_reward": 0.2576904296875,
      "kl": 0.8548611015081405,
      "learning_rate": 5e-06,
      "loss": 0.0552,
      "reward": 0.17943925857543946,
      "reward_std": 1.0328819096088409,
      "rewards/reward_func": 0.17943925857543946,
      "step": 2890,
      "toxic_reward": 3.6138275027275086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.725,
      "epoch": 0.6852551984877127,
      "format_reward": -0.5,
      "grad_norm": 8.174365997314453,
      "image_reward": 0.27861836850643157,
      "kl": 2.0340675324201585,
      "learning_rate": 5e-06,
      "loss": 0.1112,
      "reward": 0.21913965195417404,
      "reward_std": 1.3600813373923302,
      "rewards/reward_func": 0.21913965195417404,
      "step": 2900,
      "toxic_reward": 3.973111832141876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.375,
      "epoch": 0.6876181474480151,
      "format_reward": 0.0,
      "grad_norm": 9.611124992370605,
      "image_reward": 0.2759572356939316,
      "kl": 1.901711493730545,
      "learning_rate": 5e-06,
      "loss": 0.0008,
      "reward": 0.7438287258148193,
      "reward_std": 0.6283189944922924,
      "rewards/reward_func": 0.7438287258148193,
      "step": 2910,
      "toxic_reward": 3.9766014724969865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.675,
      "epoch": 0.6899810964083176,
      "format_reward": -0.75,
      "grad_norm": 4.556710243225098,
      "image_reward": 0.25573730319738386,
      "kl": 2.2221992775797843,
      "learning_rate": 5e-06,
      "loss": 0.0776,
      "reward": -0.4340919256210327,
      "reward_std": 1.778307182714343,
      "rewards/reward_func": -0.4340919256210327,
      "step": 2920,
      "toxic_reward": 4.26712441444397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.45,
      "epoch": 0.69234404536862,
      "format_reward": -0.25,
      "grad_norm": 8.245325088500977,
      "image_reward": 0.28000691831111907,
      "kl": 1.5203486174345016,
      "learning_rate": 5e-06,
      "loss": -0.0075,
      "reward": 0.38065839409828184,
      "reward_std": 1.2137143149971963,
      "rewards/reward_func": 0.38065839409828184,
      "step": 2930,
      "toxic_reward": 4.011340999603272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.95,
      "epoch": 0.6947069943289225,
      "format_reward": -0.5,
      "grad_norm": 43.48079299926758,
      "image_reward": 0.28537089079618455,
      "kl": 4.194944667816162,
      "learning_rate": 5e-06,
      "loss": -0.0192,
      "reward": -0.4992818832397461,
      "reward_std": 1.652469713240862,
      "rewards/reward_func": -0.4992818832397461,
      "step": 2940,
      "toxic_reward": 3.73269322514534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.375,
      "epoch": 0.697069943289225,
      "format_reward": -0.25,
      "grad_norm": 2.9284157752990723,
      "image_reward": 0.29124247282743454,
      "kl": 1.9233473122119904,
      "learning_rate": 5e-06,
      "loss": -0.0558,
      "reward": 0.31386570632457733,
      "reward_std": 1.3490888617932797,
      "rewards/reward_func": 0.31386570632457733,
      "step": 2950,
      "toxic_reward": 3.3832929611206053
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.9,
      "epoch": 0.6994328922495274,
      "format_reward": -0.75,
      "grad_norm": 5.489762306213379,
      "image_reward": 0.27443746030330657,
      "kl": 11.033294987678527,
      "learning_rate": 5e-06,
      "loss": 0.034,
      "reward": -0.6967712700366974,
      "reward_std": 1.7560975707136095,
      "rewards/reward_func": -0.6967712700366974,
      "step": 2960,
      "toxic_reward": 4.0662164211273195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.575,
      "epoch": 0.7017958412098299,
      "format_reward": -0.25,
      "grad_norm": 18.22649574279785,
      "image_reward": 0.26611735075712206,
      "kl": 3.9552819430828094,
      "learning_rate": 5e-06,
      "loss": -0.097,
      "reward": 0.2059646487236023,
      "reward_std": 1.4741453856229783,
      "rewards/reward_func": 0.2059646487236023,
      "step": 2970,
      "toxic_reward": 3.947977590560913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.25,
      "epoch": 0.7041587901701323,
      "format_reward": 0.0,
      "grad_norm": 17.15437889099121,
      "image_reward": 0.29809672236442564,
      "kl": 2.7566053330898286,
      "learning_rate": 5e-06,
      "loss": 0.008,
      "reward": 0.5703202053904534,
      "reward_std": 0.8202566847205162,
      "rewards/reward_func": 0.5703202053904534,
      "step": 2980,
      "toxic_reward": 3.724568712711334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.325,
      "epoch": 0.7065217391304348,
      "format_reward": -0.5,
      "grad_norm": 24.218904495239258,
      "image_reward": 0.26676025390625,
      "kl": 2.1775312602519987,
      "learning_rate": 5e-06,
      "loss": 0.0501,
      "reward": 0.04957394301891327,
      "reward_std": 1.5467448111623525,
      "rewards/reward_func": 0.04957394301891327,
      "step": 2990,
      "toxic_reward": 3.3577624768018723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.1,
      "epoch": 0.7088846880907372,
      "format_reward": -0.25,
      "grad_norm": 4.296006679534912,
      "image_reward": 0.286920166015625,
      "kl": 1.4036121606826781,
      "learning_rate": 5e-06,
      "loss": -0.0237,
      "reward": 0.3820555150508881,
      "reward_std": 1.188760439120233,
      "rewards/reward_func": 0.3820555150508881,
      "step": 3000,
      "toxic_reward": 4.305025839805603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.9,
      "epoch": 0.7112476370510397,
      "format_reward": -0.5,
      "grad_norm": 20.778005599975586,
      "image_reward": 0.30558042062653434,
      "kl": 1.4864997833967208,
      "learning_rate": 5e-06,
      "loss": 0.0741,
      "reward": -0.08508440256118774,
      "reward_std": 1.637317718565464,
      "rewards/reward_func": -0.08508440256118774,
      "step": 3010,
      "toxic_reward": 4.079210705227322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.95,
      "epoch": 0.7136105860113422,
      "format_reward": -0.25,
      "grad_norm": 4.398971080780029,
      "image_reward": 0.2982396438717842,
      "kl": 36.805122749507426,
      "learning_rate": 5e-06,
      "loss": 0.0657,
      "reward": -0.5174520492553711,
      "reward_std": 1.1937666054815055,
      "rewards/reward_func": -0.5174520492553711,
      "step": 3020,
      "toxic_reward": 4.007938003540039
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.35,
      "epoch": 0.7159735349716446,
      "format_reward": -0.5,
      "grad_norm": 13.247093200683594,
      "image_reward": 0.26703898310661317,
      "kl": 1.961264681816101,
      "learning_rate": 5e-06,
      "loss": -0.0169,
      "reward": -0.05578238368034363,
      "reward_std": 1.441930427402258,
      "rewards/reward_func": -0.05578238368034363,
      "step": 3030,
      "toxic_reward": 4.098655521869659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.275,
      "epoch": 0.718336483931947,
      "format_reward": 0.0,
      "grad_norm": 23.132400512695312,
      "image_reward": 0.2803761810064316,
      "kl": 26.18696767091751,
      "learning_rate": 5e-06,
      "loss": 0.0457,
      "reward": 0.49192982316017153,
      "reward_std": 0.6619096536189317,
      "rewards/reward_func": 0.49192982316017153,
      "step": 3040,
      "toxic_reward": 4.184990978240966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.35,
      "epoch": 0.7206994328922496,
      "format_reward": -0.25,
      "grad_norm": 28.89768409729004,
      "image_reward": 0.2594024658203125,
      "kl": 2.125564157962799,
      "learning_rate": 5e-06,
      "loss": 0.0056,
      "reward": -0.18149735927581787,
      "reward_std": 1.4747628048062325,
      "rewards/reward_func": -0.18149735927581787,
      "step": 3050,
      "toxic_reward": 3.5373760223388673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.95,
      "epoch": 0.723062381852552,
      "format_reward": -0.5,
      "grad_norm": 17.26774787902832,
      "image_reward": 0.29308573305606844,
      "kl": 13.767069751024247,
      "learning_rate": 5e-06,
      "loss": 0.0058,
      "reward": 0.02059091329574585,
      "reward_std": 1.5365628942847251,
      "rewards/reward_func": 0.02059091329574585,
      "step": 3060,
      "toxic_reward": 3.280546021461487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.65,
      "epoch": 0.7254253308128544,
      "format_reward": -0.5,
      "grad_norm": 16.69405746459961,
      "image_reward": 0.27832234650850296,
      "kl": 2.4563605159521105,
      "learning_rate": 5e-06,
      "loss": 0.0249,
      "reward": -0.4515081524848938,
      "reward_std": 1.2287155898287891,
      "rewards/reward_func": -0.4515081524848938,
      "step": 3070,
      "toxic_reward": 3.398514473438263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.775,
      "epoch": 0.7277882797731569,
      "format_reward": -0.5,
      "grad_norm": 7.214962482452393,
      "image_reward": 0.27323404848575594,
      "kl": 1.8898794114589692,
      "learning_rate": 5e-06,
      "loss": -0.0003,
      "reward": -0.4889628529548645,
      "reward_std": 1.4541106900200247,
      "rewards/reward_func": -0.4889628529548645,
      "step": 3080,
      "toxic_reward": 4.181539106369018
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.1,
      "epoch": 0.7301512287334594,
      "format_reward": -0.5,
      "grad_norm": 1.7761129140853882,
      "image_reward": 0.27235768735408783,
      "kl": 2.1771215945482254,
      "learning_rate": 5e-06,
      "loss": -0.0612,
      "reward": 0.1671779692173004,
      "reward_std": 1.4098370391875505,
      "rewards/reward_func": 0.1671779692173004,
      "step": 3090,
      "toxic_reward": 3.8193355441093444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.85,
      "epoch": 0.7325141776937618,
      "format_reward": -0.5,
      "grad_norm": 5.988401412963867,
      "image_reward": 0.25819803923368456,
      "kl": 0.8303129658102989,
      "learning_rate": 5e-06,
      "loss": -0.0141,
      "reward": 0.20456358194351196,
      "reward_std": 1.755793434381485,
      "rewards/reward_func": 0.20456358194351196,
      "step": 3100,
      "toxic_reward": 3.5276977360248565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.825,
      "epoch": 0.7348771266540642,
      "format_reward": 0.0,
      "grad_norm": 63.649696350097656,
      "image_reward": 0.26085103303194046,
      "kl": 2.509291835129261,
      "learning_rate": 5e-06,
      "loss": -0.0945,
      "reward": 0.34231345951557157,
      "reward_std": 1.2596007108688354,
      "rewards/reward_func": 0.34231345951557157,
      "step": 3110,
      "toxic_reward": 3.680406093597412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.675,
      "epoch": 0.7372400756143668,
      "format_reward": 0.0,
      "grad_norm": 13.457945823669434,
      "image_reward": 0.2661163330078125,
      "kl": 3.3423233568668365,
      "learning_rate": 5e-06,
      "loss": -0.0837,
      "reward": 0.21805171072483062,
      "reward_std": 1.0620483674108983,
      "rewards/reward_func": 0.21805171072483062,
      "step": 3120,
      "toxic_reward": 3.4958622455596924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.25,
      "epoch": 0.7396030245746692,
      "format_reward": 0.0,
      "grad_norm": 1.3712886571884155,
      "image_reward": 0.280279541015625,
      "kl": 0.5368543028831482,
      "learning_rate": 5e-06,
      "loss": 0.0856,
      "reward": -0.08258238434791565,
      "reward_std": 0.7678581360727549,
      "rewards/reward_func": -0.08258238434791565,
      "step": 3130,
      "toxic_reward": 4.090320491790772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.75,
      "epoch": 0.7419659735349716,
      "format_reward": -0.25,
      "grad_norm": 33.164817810058594,
      "image_reward": 0.263348388671875,
      "kl": 3.852606762945652,
      "learning_rate": 5e-06,
      "loss": -0.1711,
      "reward": 0.23493566811084748,
      "reward_std": 1.2882447349838912,
      "rewards/reward_func": 0.23493566811084748,
      "step": 3140,
      "toxic_reward": 3.8201312363147735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.275,
      "epoch": 0.744328922495274,
      "format_reward": -0.25,
      "grad_norm": 23.956363677978516,
      "image_reward": 0.31206461489200593,
      "kl": 0.8646048396825791,
      "learning_rate": 5e-06,
      "loss": -0.0402,
      "reward": 0.018216264247894288,
      "reward_std": 1.226612313091755,
      "rewards/reward_func": 0.018216264247894288,
      "step": 3150,
      "toxic_reward": 3.7303581714630125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.025,
      "epoch": 0.7466918714555766,
      "format_reward": -0.25,
      "grad_norm": 7.992063999176025,
      "image_reward": 0.283380126953125,
      "kl": 1.2732116781175136,
      "learning_rate": 5e-06,
      "loss": -0.091,
      "reward": 0.7706227093935013,
      "reward_std": 1.4939947571605443,
      "rewards/reward_func": 0.7706227093935013,
      "step": 3160,
      "toxic_reward": 3.5458990573883056
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.05,
      "epoch": 0.749054820415879,
      "format_reward": 0.0,
      "grad_norm": 26.938879013061523,
      "image_reward": 0.2929168701171875,
      "kl": 4.621248189732432,
      "learning_rate": 5e-06,
      "loss": -0.0543,
      "reward": 0.26634013652801514,
      "reward_std": 0.6591222167015076,
      "rewards/reward_func": 0.26634013652801514,
      "step": 3170,
      "toxic_reward": 4.090583860874176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.825,
      "epoch": 0.7514177693761814,
      "format_reward": -0.5,
      "grad_norm": 1.0445924997329712,
      "image_reward": 0.27820536196231843,
      "kl": 1.2374065339565277,
      "learning_rate": 5e-06,
      "loss": 0.0777,
      "reward": -0.1608543336391449,
      "reward_std": 0.9281521745026111,
      "rewards/reward_func": -0.1608543336391449,
      "step": 3180,
      "toxic_reward": 3.968969798088074
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.8,
      "epoch": 0.753780718336484,
      "format_reward": 0.0,
      "grad_norm": 26.366165161132812,
      "image_reward": 0.27580566257238387,
      "kl": 11.984261164069176,
      "learning_rate": 5e-06,
      "loss": 0.0176,
      "reward": 0.36088051795959475,
      "reward_std": 0.737302597053349,
      "rewards/reward_func": 0.36088051795959475,
      "step": 3190,
      "toxic_reward": 4.220689821243286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.525,
      "epoch": 0.7561436672967864,
      "format_reward": 0.0,
      "grad_norm": 15.66350269317627,
      "image_reward": 0.2542442321777344,
      "kl": 1.2004614934325217,
      "learning_rate": 5e-06,
      "loss": -0.1499,
      "reward": 0.916411966085434,
      "reward_std": 1.1410479605197907,
      "rewards/reward_func": 0.916411966085434,
      "step": 3200,
      "toxic_reward": 3.5325961112976074
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.375,
      "epoch": 0.7585066162570888,
      "format_reward": 0.0,
      "grad_norm": 4.70230770111084,
      "image_reward": 0.30909423828125,
      "kl": 2.567577276751399,
      "learning_rate": 5e-06,
      "loss": -0.0471,
      "reward": 0.20682075023651122,
      "reward_std": 0.5303860757499933,
      "rewards/reward_func": 0.20682075023651122,
      "step": 3210,
      "toxic_reward": 3.270925796031952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.05,
      "epoch": 0.7608695652173914,
      "format_reward": 0.0,
      "grad_norm": 10.79295539855957,
      "image_reward": 0.25408528596162794,
      "kl": 2.86144537627697,
      "learning_rate": 5e-06,
      "loss": -0.0379,
      "reward": 0.6601236045360566,
      "reward_std": 0.7253405870869756,
      "rewards/reward_func": 0.6601236045360566,
      "step": 3220,
      "toxic_reward": 4.3341371536254885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.725,
      "epoch": 0.7632325141776938,
      "format_reward": -0.25,
      "grad_norm": 6.788066387176514,
      "image_reward": 0.2796641021966934,
      "kl": 5.517164082825184,
      "learning_rate": 5e-06,
      "loss": 0.1356,
      "reward": -0.18339840769767762,
      "reward_std": 1.0695885993540286,
      "rewards/reward_func": -0.18339840769767762,
      "step": 3230,
      "toxic_reward": 4.077802658081055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.5,
      "epoch": 0.7655954631379962,
      "format_reward": -0.5,
      "grad_norm": 39.19500732421875,
      "image_reward": 0.2956451416015625,
      "kl": 0.7065762653946877,
      "learning_rate": 5e-06,
      "loss": -0.047,
      "reward": -0.26765223741531374,
      "reward_std": 1.6595379646867514,
      "rewards/reward_func": -0.26765223741531374,
      "step": 3240,
      "toxic_reward": 3.865544855594635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.675,
      "epoch": 0.7679584120982986,
      "format_reward": 0.0,
      "grad_norm": 8.50940990447998,
      "image_reward": 0.28647562563419343,
      "kl": 4.316986609622836,
      "learning_rate": 5e-06,
      "loss": 0.124,
      "reward": 0.6616093635559082,
      "reward_std": 1.070189495384693,
      "rewards/reward_func": 0.6616093635559082,
      "step": 3250,
      "toxic_reward": 3.284928467869759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.175,
      "epoch": 0.7703213610586012,
      "format_reward": 0.0,
      "grad_norm": 21.128314971923828,
      "image_reward": 0.2679835006594658,
      "kl": 4.375968629121781,
      "learning_rate": 5e-06,
      "loss": 0.0579,
      "reward": 0.3372311323881149,
      "reward_std": 0.869463924318552,
      "rewards/reward_func": 0.3372311323881149,
      "step": 3260,
      "toxic_reward": 3.78046395778656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.5,
      "epoch": 0.7726843100189036,
      "format_reward": 0.0,
      "grad_norm": 7.558209419250488,
      "image_reward": 0.2745330810546875,
      "kl": 2.3013378672301767,
      "learning_rate": 5e-06,
      "loss": -0.031,
      "reward": 0.784791512787342,
      "reward_std": 0.8750310368835926,
      "rewards/reward_func": 0.784791512787342,
      "step": 3270,
      "toxic_reward": 3.3566872388124467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.025,
      "epoch": 0.775047258979206,
      "format_reward": -0.5,
      "grad_norm": 2.0432510375976562,
      "image_reward": 0.2913625091314316,
      "kl": 0.3781319923698902,
      "learning_rate": 5e-06,
      "loss": -0.0607,
      "reward": -0.3740895688533783,
      "reward_std": 1.350129895284772,
      "rewards/reward_func": -0.3740895688533783,
      "step": 3280,
      "toxic_reward": 4.018161624670029
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.825,
      "epoch": 0.7774102079395085,
      "format_reward": -0.5,
      "grad_norm": 3.16352915763855,
      "image_reward": 0.27322998046875,
      "kl": 0.3104788601398468,
      "learning_rate": 5e-06,
      "loss": 0.0334,
      "reward": -0.05296646356582642,
      "reward_std": 1.2258484821766615,
      "rewards/reward_func": -0.05296646356582642,
      "step": 3290,
      "toxic_reward": 4.389449417591095
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.925,
      "epoch": 0.779773156899811,
      "format_reward": 0.0,
      "grad_norm": 4.182164669036865,
      "image_reward": 0.2737925201654434,
      "kl": 1.2850206293165685,
      "learning_rate": 5e-06,
      "loss": -0.1213,
      "reward": 0.6742017388343811,
      "reward_std": 0.736553730070591,
      "rewards/reward_func": 0.6742017388343811,
      "step": 3300,
      "toxic_reward": 4.220770263671875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.175,
      "epoch": 0.7821361058601134,
      "format_reward": -0.25,
      "grad_norm": 8.606978416442871,
      "image_reward": 0.2700215637683868,
      "kl": 4.289887800067663,
      "learning_rate": 5e-06,
      "loss": -0.1432,
      "reward": 0.30965389013290406,
      "reward_std": 1.063696064054966,
      "rewards/reward_func": 0.30965389013290406,
      "step": 3310,
      "toxic_reward": 4.136632585525513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.45,
      "epoch": 0.7844990548204159,
      "format_reward": 0.0,
      "grad_norm": 3.973367691040039,
      "image_reward": 0.25255330502986906,
      "kl": 6.628417156636715,
      "learning_rate": 5e-06,
      "loss": -0.0524,
      "reward": 0.2595418691635132,
      "reward_std": 0.6656439051032066,
      "rewards/reward_func": 0.2595418691635132,
      "step": 3320,
      "toxic_reward": 3.946825695037842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.625,
      "epoch": 0.7868620037807184,
      "format_reward": 0.0,
      "grad_norm": 5.275523662567139,
      "image_reward": 0.29005940854549406,
      "kl": 25.900663439184427,
      "learning_rate": 5e-06,
      "loss": -0.1344,
      "reward": 0.8005503177642822,
      "reward_std": 0.9713124742731452,
      "rewards/reward_func": 0.8005503177642822,
      "step": 3330,
      "toxic_reward": 4.047469854354858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.55,
      "epoch": 0.7892249527410208,
      "format_reward": 0.0,
      "grad_norm": 5.920967102050781,
      "image_reward": 0.2706329345703125,
      "kl": 2.892443811520934,
      "learning_rate": 5e-06,
      "loss": -0.028,
      "reward": 0.7794641971588134,
      "reward_std": 0.7315312433987856,
      "rewards/reward_func": 0.7794641971588134,
      "step": 3340,
      "toxic_reward": 4.036288380622864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.325,
      "epoch": 0.7915879017013232,
      "format_reward": -0.25,
      "grad_norm": 19.411304473876953,
      "image_reward": 0.2590001419186592,
      "kl": 0.762314885109663,
      "learning_rate": 5e-06,
      "loss": 0.0738,
      "reward": -0.22335948944091796,
      "reward_std": 1.229094560444355,
      "rewards/reward_func": -0.22335948944091796,
      "step": 3350,
      "toxic_reward": 4.078046441078186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.825,
      "epoch": 0.7939508506616257,
      "format_reward": -0.25,
      "grad_norm": 9.397270202636719,
      "image_reward": 0.25453638202614254,
      "kl": 1.7116897955536843,
      "learning_rate": 5e-06,
      "loss": -0.0637,
      "reward": -0.23146066069602966,
      "reward_std": 1.4809592371806501,
      "rewards/reward_func": -0.23146066069602966,
      "step": 3360,
      "toxic_reward": 3.7261215580834284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.075,
      "epoch": 0.7963137996219282,
      "format_reward": 0.0,
      "grad_norm": 2.7069385051727295,
      "image_reward": 0.29098663330078123,
      "kl": 0.2713630013167858,
      "learning_rate": 5e-06,
      "loss": -0.0331,
      "reward": 0.2162942558526993,
      "reward_std": 0.7098794117569923,
      "rewards/reward_func": 0.2162942558526993,
      "step": 3370,
      "toxic_reward": 3.5313488602638246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.425,
      "epoch": 0.7986767485822306,
      "format_reward": 0.0,
      "grad_norm": 5.024960041046143,
      "image_reward": 0.2739929184317589,
      "kl": 18.83201899640262,
      "learning_rate": 5e-06,
      "loss": -0.0017,
      "reward": 0.4777979046106339,
      "reward_std": 1.240721021965146,
      "rewards/reward_func": 0.4777979046106339,
      "step": 3380,
      "toxic_reward": 3.522536587715149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.725,
      "epoch": 0.8010396975425331,
      "format_reward": 0.0,
      "grad_norm": 4.2769622802734375,
      "image_reward": 0.2533442169427872,
      "kl": 8.165257753431797,
      "learning_rate": 5e-06,
      "loss": -0.0776,
      "reward": 0.44249573945999143,
      "reward_std": 0.8017176885157824,
      "rewards/reward_func": 0.44249573945999143,
      "step": 3390,
      "toxic_reward": 4.304618096351623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.85,
      "epoch": 0.8034026465028355,
      "format_reward": -0.75,
      "grad_norm": 6.779478549957275,
      "image_reward": 0.254600016772747,
      "kl": 3.2404680982232095,
      "learning_rate": 5e-06,
      "loss": -0.0622,
      "reward": 1.1459296941757202,
      "reward_std": 2.3130407273769378,
      "rewards/reward_func": 1.1459296941757202,
      "step": 3400,
      "toxic_reward": 3.946528363227844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.475,
      "epoch": 0.805765595463138,
      "format_reward": -0.75,
      "grad_norm": 9.082526206970215,
      "image_reward": 0.2917332977056503,
      "kl": 1.1568331263959408,
      "learning_rate": 5e-06,
      "loss": -0.087,
      "reward": -0.22108137607574463,
      "reward_std": 2.109957142919302,
      "rewards/reward_func": -0.22108137607574463,
      "step": 3410,
      "toxic_reward": 3.467973506450653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.225,
      "epoch": 0.8081285444234405,
      "format_reward": 0.0,
      "grad_norm": 4.748640537261963,
      "image_reward": 0.2629201263189316,
      "kl": 1.277670707181096,
      "learning_rate": 5e-06,
      "loss": 0.0912,
      "reward": 0.5041390061378479,
      "reward_std": 1.1238155417144298,
      "rewards/reward_func": 0.5041390061378479,
      "step": 3420,
      "toxic_reward": 3.6773669004440306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.875,
      "epoch": 0.8104914933837429,
      "format_reward": -0.25,
      "grad_norm": 6.2929182052612305,
      "image_reward": 0.2664311736822128,
      "kl": 0.41112807476893065,
      "learning_rate": 5e-06,
      "loss": -0.0185,
      "reward": 0.25773588865995406,
      "reward_std": 1.1975380808115006,
      "rewards/reward_func": 0.25773588865995406,
      "step": 3430,
      "toxic_reward": 3.5475049674510957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 31.2,
      "epoch": 0.8128544423440454,
      "format_reward": 0.0,
      "grad_norm": 2.984248399734497,
      "image_reward": 0.29365132600069044,
      "kl": 6.331273209676146,
      "learning_rate": 5e-06,
      "loss": -0.0094,
      "reward": -0.1625719666481018,
      "reward_std": 0.864103776961565,
      "rewards/reward_func": -0.1625719666481018,
      "step": 3440,
      "toxic_reward": 3.899219441413879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 59.075,
      "epoch": 0.8152173913043478,
      "format_reward": 0.0,
      "grad_norm": 5.733253479003906,
      "image_reward": 0.28086344301700594,
      "kl": 9.025772982649505,
      "learning_rate": 5e-06,
      "loss": -0.0245,
      "reward": 0.4977319598197937,
      "reward_std": 0.6485220491886139,
      "rewards/reward_func": 0.4977319598197937,
      "step": 3450,
      "toxic_reward": 3.7944631457328795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.25,
      "epoch": 0.8175803402646503,
      "format_reward": -0.25,
      "grad_norm": 5.605562686920166,
      "image_reward": 0.281341552734375,
      "kl": 0.665616973862052,
      "learning_rate": 5e-06,
      "loss": -0.0068,
      "reward": -0.26609439849853517,
      "reward_std": 1.4688232390210032,
      "rewards/reward_func": -0.26609439849853517,
      "step": 3460,
      "toxic_reward": 3.4837970972061156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.475,
      "epoch": 0.8199432892249527,
      "format_reward": -0.5,
      "grad_norm": 2.6239664554595947,
      "image_reward": 0.2684331268072128,
      "kl": 1.5078430883586407,
      "learning_rate": 5e-06,
      "loss": 0.1555,
      "reward": -0.12674018144607543,
      "reward_std": 1.4365263484418391,
      "rewards/reward_func": -0.12674018144607543,
      "step": 3470,
      "toxic_reward": 4.5540220737457275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.975,
      "epoch": 0.8223062381852552,
      "format_reward": -0.25,
      "grad_norm": 8.734126091003418,
      "image_reward": 0.2604085296392441,
      "kl": 24.937382932007313,
      "learning_rate": 5e-06,
      "loss": -0.0011,
      "reward": -0.20797204971313477,
      "reward_std": 0.9569237198680639,
      "rewards/reward_func": -0.20797204971313477,
      "step": 3480,
      "toxic_reward": 4.407214689254761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 60.625,
      "epoch": 0.8246691871455577,
      "format_reward": -0.25,
      "grad_norm": 1.5907628536224365,
      "image_reward": 0.2842885345220566,
      "kl": 0.051335761044174436,
      "learning_rate": 5e-06,
      "loss": 0.0047,
      "reward": 0.9079252362251282,
      "reward_std": 1.1933536015450954,
      "rewards/reward_func": 0.9079252362251282,
      "step": 3490,
      "toxic_reward": 4.199088740348816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.75,
      "epoch": 0.8270321361058601,
      "format_reward": -1.0,
      "grad_norm": 1.5189018249511719,
      "image_reward": 0.2938863128423691,
      "kl": 6.8780351031571625,
      "learning_rate": 5e-06,
      "loss": 0.0792,
      "reward": -0.6177874624729156,
      "reward_std": 1.9702259879559278,
      "rewards/reward_func": -0.6177874624729156,
      "step": 3500,
      "toxic_reward": 3.7184417486190795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.075,
      "epoch": 0.8293950850661626,
      "format_reward": -0.25,
      "grad_norm": 0.5691888928413391,
      "image_reward": 0.2792378753423691,
      "kl": 0.0625603836029768,
      "learning_rate": 5e-06,
      "loss": -0.0245,
      "reward": 0.39415156543254853,
      "reward_std": 0.7689090168103576,
      "rewards/reward_func": 0.39415156543254853,
      "step": 3510,
      "toxic_reward": 4.210167169570923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.675,
      "epoch": 0.831758034026465,
      "format_reward": 0.0,
      "grad_norm": 2.8700907230377197,
      "image_reward": 0.263861083984375,
      "kl": 0.3225065166130662,
      "learning_rate": 5e-06,
      "loss": -0.0609,
      "reward": 0.6327012300491333,
      "reward_std": 0.980434575676918,
      "rewards/reward_func": 0.6327012300491333,
      "step": 3520,
      "toxic_reward": 3.8261560261249543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.05,
      "epoch": 0.8341209829867675,
      "format_reward": 0.0,
      "grad_norm": 1.0346537828445435,
      "image_reward": 0.284771728515625,
      "kl": 0.40898411339148877,
      "learning_rate": 5e-06,
      "loss": -0.0307,
      "reward": 0.30759164690971375,
      "reward_std": 0.6451162457466125,
      "rewards/reward_func": 0.30759164690971375,
      "step": 3530,
      "toxic_reward": 4.171144628524781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.3,
      "epoch": 0.8364839319470699,
      "format_reward": -0.5,
      "grad_norm": 5.951425075531006,
      "image_reward": 0.3035013824701309,
      "kl": 11.781208837591112,
      "learning_rate": 5e-06,
      "loss": 0.0257,
      "reward": -0.1725111722946167,
      "reward_std": 1.8102335507050156,
      "rewards/reward_func": -0.1725111722946167,
      "step": 3540,
      "toxic_reward": 3.670738685131073
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.975,
      "epoch": 0.8388468809073724,
      "format_reward": 0.0,
      "grad_norm": 14.631609916687012,
      "image_reward": 0.2706085205078125,
      "kl": 4.56192576661706,
      "learning_rate": 5e-06,
      "loss": -0.033,
      "reward": 1.4811566695570946,
      "reward_std": 0.9509499605745078,
      "rewards/reward_func": 1.4811566695570946,
      "step": 3550,
      "toxic_reward": 3.4709715723991392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.225,
      "epoch": 0.8412098298676749,
      "format_reward": -0.5,
      "grad_norm": 7.567039489746094,
      "image_reward": 0.2595652252435684,
      "kl": 10.069495621696115,
      "learning_rate": 5e-06,
      "loss": -0.057,
      "reward": 0.034914278984069826,
      "reward_std": 2.0578875496983526,
      "rewards/reward_func": 0.034914278984069826,
      "step": 3560,
      "toxic_reward": 3.711397814750671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.9,
      "epoch": 0.8435727788279773,
      "format_reward": 0.0,
      "grad_norm": 2.5686023235321045,
      "image_reward": 0.27446797788143157,
      "kl": 1.5964453139342367,
      "learning_rate": 5e-06,
      "loss": -0.031,
      "reward": 0.41478089690208436,
      "reward_std": 0.6242949636653066,
      "rewards/reward_func": 0.41478089690208436,
      "step": 3570,
      "toxic_reward": 4.057631134986877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.725,
      "epoch": 0.8459357277882797,
      "format_reward": -0.75,
      "grad_norm": 18.62441062927246,
      "image_reward": 0.26212361752986907,
      "kl": 18.3546858407557,
      "learning_rate": 5e-06,
      "loss": -0.0125,
      "reward": 0.3845840930938721,
      "reward_std": 2.4349112689495085,
      "rewards/reward_func": 0.3845840930938721,
      "step": 3580,
      "toxic_reward": 3.7492689728736877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.15,
      "epoch": 0.8482986767485823,
      "format_reward": -0.25,
      "grad_norm": 1.5018891096115112,
      "image_reward": 0.25630950927734375,
      "kl": 1.5399845570325852,
      "learning_rate": 5e-06,
      "loss": 0.0064,
      "reward": -0.4008509755134583,
      "reward_std": 1.2334194054827095,
      "rewards/reward_func": -0.4008509755134583,
      "step": 3590,
      "toxic_reward": 3.9793298959732057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.475,
      "epoch": 0.8506616257088847,
      "format_reward": 0.0,
      "grad_norm": 1.118828296661377,
      "image_reward": 0.279705810546875,
      "kl": 2.2166069228202105,
      "learning_rate": 5e-06,
      "loss": -0.0439,
      "reward": 0.722964608669281,
      "reward_std": 0.7349236082285643,
      "rewards/reward_func": 0.722964608669281,
      "step": 3600,
      "toxic_reward": 4.429630327224731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.8,
      "epoch": 0.8530245746691871,
      "format_reward": 0.0,
      "grad_norm": 2.9072647094726562,
      "image_reward": 0.2988444000482559,
      "kl": 0.4259108882397413,
      "learning_rate": 5e-06,
      "loss": -0.139,
      "reward": 0.36530678868293764,
      "reward_std": 1.0169117324054242,
      "rewards/reward_func": 0.36530678868293764,
      "step": 3610,
      "toxic_reward": 3.706578254699707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.05,
      "epoch": 0.8553875236294896,
      "format_reward": 0.0,
      "grad_norm": 5.574492931365967,
      "image_reward": 0.26971537321805955,
      "kl": 0.8386783060617745,
      "learning_rate": 5e-06,
      "loss": 0.0196,
      "reward": -0.043644605576992034,
      "reward_std": 0.7492304600775241,
      "rewards/reward_func": -0.043644605576992034,
      "step": 3620,
      "toxic_reward": 3.7889950960874557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.2,
      "epoch": 0.8577504725897921,
      "format_reward": 0.0,
      "grad_norm": 0.6372764110565186,
      "image_reward": 0.2799346923828125,
      "kl": 0.09604998417198658,
      "learning_rate": 5e-06,
      "loss": -0.0245,
      "reward": 0.31555656492710116,
      "reward_std": 0.5240693692117929,
      "rewards/reward_func": 0.31555656492710116,
      "step": 3630,
      "toxic_reward": 3.9185105204582213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.775,
      "epoch": 0.8601134215500945,
      "format_reward": 0.0,
      "grad_norm": 1.1808196306228638,
      "image_reward": 0.2841166198253632,
      "kl": 29.371498390100896,
      "learning_rate": 5e-06,
      "loss": -0.0516,
      "reward": 0.26911270916461943,
      "reward_std": 0.5647319633513689,
      "rewards/reward_func": 0.26911270916461943,
      "step": 3640,
      "toxic_reward": 3.410293960571289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.35,
      "epoch": 0.8624763705103969,
      "format_reward": -0.25,
      "grad_norm": 0.7336105108261108,
      "image_reward": 0.26638386994600294,
      "kl": 2.7957767372950912,
      "learning_rate": 5e-06,
      "loss": 0.0919,
      "reward": 0.25321381688117983,
      "reward_std": 1.543316999450326,
      "rewards/reward_func": 0.25321381688117983,
      "step": 3650,
      "toxic_reward": 3.6566759824752806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.75,
      "epoch": 0.8648393194706995,
      "format_reward": -0.25,
      "grad_norm": 0.6029968857765198,
      "image_reward": 0.2995513916015625,
      "kl": 2.5597430652938784,
      "learning_rate": 5e-06,
      "loss": -0.0974,
      "reward": 0.32535398602485655,
      "reward_std": 1.3309460416436196,
      "rewards/reward_func": 0.32535398602485655,
      "step": 3660,
      "toxic_reward": 3.65915470123291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.275,
      "epoch": 0.8672022684310019,
      "format_reward": -0.75,
      "grad_norm": 3.770862102508545,
      "image_reward": 0.291839599609375,
      "kl": 24.679713291302324,
      "learning_rate": 5e-06,
      "loss": -0.0464,
      "reward": -0.6877359867095947,
      "reward_std": 2.206609180383384,
      "rewards/reward_func": -0.6877359867095947,
      "step": 3670,
      "toxic_reward": 3.764638936519623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.9,
      "epoch": 0.8695652173913043,
      "format_reward": -0.25,
      "grad_norm": 0.5417113304138184,
      "image_reward": 0.26164347380399705,
      "kl": 0.35583615899085996,
      "learning_rate": 5e-06,
      "loss": 0.0668,
      "reward": -0.214414319396019,
      "reward_std": 1.3576272014528512,
      "rewards/reward_func": -0.214414319396019,
      "step": 3680,
      "toxic_reward": 3.65915904045105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.125,
      "epoch": 0.8719281663516069,
      "format_reward": 0.0,
      "grad_norm": 0.4714978039264679,
      "image_reward": 0.278389485180378,
      "kl": 2.4852739069610834,
      "learning_rate": 5e-06,
      "loss": -0.0177,
      "reward": 0.1318028151988983,
      "reward_std": 0.8923286706209183,
      "rewards/reward_func": 0.1318028151988983,
      "step": 3690,
      "toxic_reward": 3.508782708644867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.05,
      "epoch": 0.8742911153119093,
      "format_reward": 0.0,
      "grad_norm": 0.8036189079284668,
      "image_reward": 0.246942138671875,
      "kl": 17.79970283471048,
      "learning_rate": 5e-06,
      "loss": -0.0525,
      "reward": 0.44554237723350526,
      "reward_std": 0.8977296775206923,
      "rewards/reward_func": 0.44554237723350526,
      "step": 3700,
      "toxic_reward": 3.5204819679260253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.15,
      "epoch": 0.8766540642722117,
      "format_reward": 0.0,
      "grad_norm": 0.9322050213813782,
      "image_reward": 0.2607330322265625,
      "kl": 4.579995289538056,
      "learning_rate": 5e-06,
      "loss": -0.0918,
      "reward": 0.7257406830787658,
      "reward_std": 0.7061707813292741,
      "rewards/reward_func": 0.7257406830787658,
      "step": 3710,
      "toxic_reward": 3.967698335647583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 32.9,
      "epoch": 0.8790170132325141,
      "format_reward": -0.5,
      "grad_norm": 0.7602401971817017,
      "image_reward": 0.277716064453125,
      "kl": 5.202583113871515,
      "learning_rate": 5e-06,
      "loss": 0.0093,
      "reward": -0.1386810451745987,
      "reward_std": 1.5367558933794498,
      "rewards/reward_func": -0.1386810451745987,
      "step": 3720,
      "toxic_reward": 3.343963861465454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.45,
      "epoch": 0.8813799621928167,
      "format_reward": 0.0,
      "grad_norm": 0.49207255244255066,
      "image_reward": 0.2617726638913155,
      "kl": 3.0696171432733537,
      "learning_rate": 5e-06,
      "loss": 0.0096,
      "reward": 0.8290068447589874,
      "reward_std": 0.6912821188569069,
      "rewards/reward_func": 0.8290068447589874,
      "step": 3730,
      "toxic_reward": 4.308743190765381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.125,
      "epoch": 0.8837429111531191,
      "format_reward": 0.0,
      "grad_norm": 0.5754015445709229,
      "image_reward": 0.25755615234375,
      "kl": 1.463007004186511,
      "learning_rate": 5e-06,
      "loss": -0.074,
      "reward": 1.1725465416908265,
      "reward_std": 0.7939416155219078,
      "rewards/reward_func": 1.1725465416908265,
      "step": 3740,
      "toxic_reward": 3.892818683385849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.65,
      "epoch": 0.8861058601134215,
      "format_reward": -0.5,
      "grad_norm": 0.3917323350906372,
      "image_reward": 0.2610259994864464,
      "kl": 3.9046508548781276,
      "learning_rate": 5e-06,
      "loss": -0.0114,
      "reward": 0.1690664052963257,
      "reward_std": 1.9762837937101723,
      "rewards/reward_func": 0.1690664052963257,
      "step": 3750,
      "toxic_reward": 3.7723870635032655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.45,
      "epoch": 0.888468809073724,
      "format_reward": 0.0,
      "grad_norm": 0.6322398781776428,
      "image_reward": 0.27677764892578127,
      "kl": 1.8196211833506823,
      "learning_rate": 5e-06,
      "loss": -0.12,
      "reward": 0.42850649207830427,
      "reward_std": 0.5486618679948151,
      "rewards/reward_func": 0.42850649207830427,
      "step": 3760,
      "toxic_reward": 3.4346215546131136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.75,
      "epoch": 0.8908317580340265,
      "format_reward": 0.0,
      "grad_norm": 0.3245849013328552,
      "image_reward": 0.29166819155216217,
      "kl": 10.705555348284543,
      "learning_rate": 5e-06,
      "loss": -0.0603,
      "reward": 0.061820387840270996,
      "reward_std": 1.08290204256773,
      "rewards/reward_func": 0.061820387840270996,
      "step": 3770,
      "toxic_reward": 2.949862742424011
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.0,
      "epoch": 0.8931947069943289,
      "format_reward": -0.25,
      "grad_norm": 0.3298509418964386,
      "image_reward": 0.290167236328125,
      "kl": 0.07300702948123217,
      "learning_rate": 5e-06,
      "loss": -0.0171,
      "reward": 0.06625822186470032,
      "reward_std": 1.0081432062666864,
      "rewards/reward_func": 0.06625822186470032,
      "step": 3780,
      "toxic_reward": 4.22382138967514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.4,
      "epoch": 0.8955576559546313,
      "format_reward": 0.0,
      "grad_norm": 0.698654055595398,
      "image_reward": 0.27091064453125,
      "kl": 4.801618622988462,
      "learning_rate": 5e-06,
      "loss": -0.0591,
      "reward": 0.3187494039535522,
      "reward_std": 0.5140533071011305,
      "rewards/reward_func": 0.3187494039535522,
      "step": 3790,
      "toxic_reward": 4.416417121887207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.6,
      "epoch": 0.8979206049149339,
      "format_reward": -0.25,
      "grad_norm": 0.6394158601760864,
      "image_reward": 0.26355692744255066,
      "kl": 3.265846297331154,
      "learning_rate": 5e-06,
      "loss": -0.0384,
      "reward": -0.14046210050582886,
      "reward_std": 1.0342714745551347,
      "rewards/reward_func": -0.14046210050582886,
      "step": 3800,
      "toxic_reward": 4.3116097211837765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.7,
      "epoch": 0.9002835538752363,
      "format_reward": -0.5,
      "grad_norm": 0.7541901469230652,
      "image_reward": 0.2673909515142441,
      "kl": 0.7993329163640738,
      "learning_rate": 5e-06,
      "loss": 0.0777,
      "reward": 0.010242342948913574,
      "reward_std": 1.4442682154476643,
      "rewards/reward_func": 0.010242342948913574,
      "step": 3810,
      "toxic_reward": 4.425883173942566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.225,
      "epoch": 0.9026465028355387,
      "format_reward": 0.0,
      "grad_norm": 0.8831507563591003,
      "image_reward": 0.29705912470817564,
      "kl": 3.6087327402085068,
      "learning_rate": 5e-06,
      "loss": -0.111,
      "reward": 0.8021630614995956,
      "reward_std": 0.8431573905050754,
      "rewards/reward_func": 0.8021630614995956,
      "step": 3820,
      "toxic_reward": 3.6668890714645386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.85,
      "epoch": 0.9050094517958412,
      "format_reward": 0.0,
      "grad_norm": 1.166309118270874,
      "image_reward": 0.27442220151424407,
      "kl": 3.696834401600063,
      "learning_rate": 5e-06,
      "loss": 0.023,
      "reward": 0.46357709765434263,
      "reward_std": 0.5384013399481773,
      "rewards/reward_func": 0.46357709765434263,
      "step": 3830,
      "toxic_reward": 4.282819819450379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.275,
      "epoch": 0.9073724007561437,
      "format_reward": -0.25,
      "grad_norm": 2.2214293479919434,
      "image_reward": 0.29029541015625,
      "kl": 6.355313093215227,
      "learning_rate": 5e-06,
      "loss": 0.0587,
      "reward": 0.36757221817970276,
      "reward_std": 1.1468286462128163,
      "rewards/reward_func": 0.36757221817970276,
      "step": 3840,
      "toxic_reward": 3.8713893949985505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.525,
      "epoch": 0.9097353497164461,
      "format_reward": 0.0,
      "grad_norm": 0.7023747563362122,
      "image_reward": 0.2795267730951309,
      "kl": 0.12285411208868027,
      "learning_rate": 5e-06,
      "loss": 0.0277,
      "reward": 0.6907171040773392,
      "reward_std": 0.8528184913098812,
      "rewards/reward_func": 0.6907171040773392,
      "step": 3850,
      "toxic_reward": 3.9646514534950255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.0,
      "epoch": 0.9120982986767486,
      "format_reward": -0.25,
      "grad_norm": 0.6574695706367493,
      "image_reward": 0.277626545727253,
      "kl": 3.223006421420723,
      "learning_rate": 5e-06,
      "loss": 0.1019,
      "reward": 0.17425565123558046,
      "reward_std": 1.0788604862987996,
      "rewards/reward_func": 0.17425565123558046,
      "step": 3860,
      "toxic_reward": 3.892214322090149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.575,
      "epoch": 0.9144612476370511,
      "format_reward": 0.0,
      "grad_norm": 1.6060093641281128,
      "image_reward": 0.25700276643037795,
      "kl": 1.728565347008407,
      "learning_rate": 5e-06,
      "loss": -0.0095,
      "reward": 0.2703657388687134,
      "reward_std": 0.8089243900030851,
      "rewards/reward_func": 0.2703657388687134,
      "step": 3870,
      "toxic_reward": 4.175320339202881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.275,
      "epoch": 0.9168241965973535,
      "format_reward": 0.0,
      "grad_norm": 2.603025436401367,
      "image_reward": 0.2737335205078125,
      "kl": 0.7518249765969813,
      "learning_rate": 5e-06,
      "loss": -0.0306,
      "reward": 0.8955561727285385,
      "reward_std": 1.1253668650984765,
      "rewards/reward_func": 0.8955561727285385,
      "step": 3880,
      "toxic_reward": 3.5497735261917116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.875,
      "epoch": 0.9191871455576559,
      "format_reward": -0.25,
      "grad_norm": 0.6174436211585999,
      "image_reward": 0.2833251953125,
      "kl": 1.3900917531922459,
      "learning_rate": 5e-06,
      "loss": 0.0533,
      "reward": 0.8824085891246796,
      "reward_std": 1.2487390112131833,
      "rewards/reward_func": 0.8824085891246796,
      "step": 3890,
      "toxic_reward": 3.764987659454346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.1,
      "epoch": 0.9215500945179584,
      "format_reward": -0.5,
      "grad_norm": 0.8587064146995544,
      "image_reward": 0.25449015349149706,
      "kl": 3.2844431857578456,
      "learning_rate": 5e-06,
      "loss": 0.0836,
      "reward": 0.17285645604133607,
      "reward_std": 1.4729075387120247,
      "rewards/reward_func": 0.17285645604133607,
      "step": 3900,
      "toxic_reward": 4.319640278816223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.65,
      "epoch": 0.9239130434782609,
      "format_reward": -0.25,
      "grad_norm": 0.7836766242980957,
      "image_reward": 0.2760904937982559,
      "kl": 0.04128519091755152,
      "learning_rate": 5e-06,
      "loss": 0.0218,
      "reward": -0.14393893480300904,
      "reward_std": 1.2086152411997317,
      "rewards/reward_func": -0.14393893480300904,
      "step": 3910,
      "toxic_reward": 3.988687515258789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.525,
      "epoch": 0.9262759924385633,
      "format_reward": 0.0,
      "grad_norm": 1.0223326683044434,
      "image_reward": 0.237677001953125,
      "kl": 0.10622669160366058,
      "learning_rate": 5e-06,
      "loss": 0.1241,
      "reward": 0.8052110552787781,
      "reward_std": 0.809264022950083,
      "rewards/reward_func": 0.8052110552787781,
      "step": 3920,
      "toxic_reward": 4.316140675544739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.525,
      "epoch": 0.9286389413988658,
      "format_reward": -0.25,
      "grad_norm": 1.2948088645935059,
      "image_reward": 0.27791646122932434,
      "kl": 2.2056565455161037,
      "learning_rate": 5e-06,
      "loss": 0.0904,
      "reward": 0.5610605776309967,
      "reward_std": 0.9484948962926865,
      "rewards/reward_func": 0.5610605776309967,
      "step": 3930,
      "toxic_reward": 4.4695143699646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.825,
      "epoch": 0.9310018903591682,
      "format_reward": 0.0,
      "grad_norm": 1.0040950775146484,
      "image_reward": 0.27231852263212203,
      "kl": 2.655760496482253,
      "learning_rate": 5e-06,
      "loss": -0.0755,
      "reward": 0.263138085603714,
      "reward_std": 0.4817726358771324,
      "rewards/reward_func": 0.263138085603714,
      "step": 3940,
      "toxic_reward": 4.636347913742066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.25,
      "epoch": 0.9333648393194707,
      "format_reward": -0.25,
      "grad_norm": 0.39709585905075073,
      "image_reward": 0.2637420654296875,
      "kl": 0.1439337281510234,
      "learning_rate": 5e-06,
      "loss": -0.0772,
      "reward": 0.04962950348854065,
      "reward_std": 0.781620041653514,
      "rewards/reward_func": 0.04962950348854065,
      "step": 3950,
      "toxic_reward": 4.751176500320435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.9,
      "epoch": 0.9357277882797732,
      "format_reward": -0.25,
      "grad_norm": 0.8190930485725403,
      "image_reward": 0.2555938705801964,
      "kl": 5.330091013200581,
      "learning_rate": 5e-06,
      "loss": -0.0093,
      "reward": -0.16106579303741456,
      "reward_std": 1.2331121437251569,
      "rewards/reward_func": -0.16106579303741456,
      "step": 3960,
      "toxic_reward": 4.007374119758606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.75,
      "epoch": 0.9380907372400756,
      "format_reward": 0.0,
      "grad_norm": 1.0821632146835327,
      "image_reward": 0.3061696380376816,
      "kl": 6.141950584948063,
      "learning_rate": 5e-06,
      "loss": 0.1307,
      "reward": 0.625621622800827,
      "reward_std": 0.8008190289139747,
      "rewards/reward_func": 0.625621622800827,
      "step": 3970,
      "toxic_reward": 3.468269979953766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.625,
      "epoch": 0.9404536862003781,
      "format_reward": -0.75,
      "grad_norm": 1.1129677295684814,
      "image_reward": 0.26193033903837204,
      "kl": 0.6634119726717472,
      "learning_rate": 5e-06,
      "loss": -0.0429,
      "reward": 0.13726072907447814,
      "reward_std": 2.353568767011166,
      "rewards/reward_func": 0.13726072907447814,
      "step": 3980,
      "toxic_reward": 4.0013970851898195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.35,
      "epoch": 0.9428166351606805,
      "format_reward": 0.0,
      "grad_norm": 0.7701426148414612,
      "image_reward": 0.28350016176700593,
      "kl": 0.1994122840464115,
      "learning_rate": 5e-06,
      "loss": -0.0421,
      "reward": 0.7691244065761567,
      "reward_std": 0.9025557667016983,
      "rewards/reward_func": 0.7691244065761567,
      "step": 3990,
      "toxic_reward": 4.340523219108581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.225,
      "epoch": 0.945179584120983,
      "format_reward": 0.0,
      "grad_norm": 0.46611157059669495,
      "image_reward": 0.26962890625,
      "kl": 0.047311073541641234,
      "learning_rate": 5e-06,
      "loss": -0.1003,
      "reward": 1.3654770731925965,
      "reward_std": 0.657595872040838,
      "rewards/reward_func": 1.3654770731925965,
      "step": 4000,
      "toxic_reward": 3.765986955165863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.675,
      "epoch": 0.9475425330812854,
      "format_reward": -0.25,
      "grad_norm": 0.730478048324585,
      "image_reward": 0.2595326751470566,
      "kl": 2.0693125385791062,
      "learning_rate": 5e-06,
      "loss": -0.0531,
      "reward": 0.16633399724960327,
      "reward_std": 1.2444878976792098,
      "rewards/reward_func": 0.16633399724960327,
      "step": 4010,
      "toxic_reward": 3.9091518998146055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.775,
      "epoch": 0.9499054820415879,
      "format_reward": -0.25,
      "grad_norm": 0.7307797074317932,
      "image_reward": 0.2784047439694405,
      "kl": 1.5403530787676574,
      "learning_rate": 5e-06,
      "loss": 0.01,
      "reward": 0.15964727997779846,
      "reward_std": 1.2297844395041466,
      "rewards/reward_func": 0.15964727997779846,
      "step": 4020,
      "toxic_reward": 4.325857400894165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.275,
      "epoch": 0.9522684310018904,
      "format_reward": -0.5,
      "grad_norm": 1.158098816871643,
      "image_reward": 0.240879312902689,
      "kl": 1.8537536807358266,
      "learning_rate": 5e-06,
      "loss": 0.1782,
      "reward": 0.5329648047685623,
      "reward_std": 1.5547814331948757,
      "rewards/reward_func": 0.5329648047685623,
      "step": 4030,
      "toxic_reward": 3.8254613667726516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.95,
      "epoch": 0.9546313799621928,
      "format_reward": 0.0,
      "grad_norm": 0.5303730964660645,
      "image_reward": 0.25118484497070315,
      "kl": 0.187329238653183,
      "learning_rate": 5e-06,
      "loss": 0.0541,
      "reward": 0.27067047357559204,
      "reward_std": 0.7333962991833687,
      "rewards/reward_func": 0.27067047357559204,
      "step": 4040,
      "toxic_reward": 4.245214033126831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.5,
      "epoch": 0.9569943289224953,
      "format_reward": -0.25,
      "grad_norm": 0.8333770632743835,
      "image_reward": 0.264398193359375,
      "kl": 7.662982761859894,
      "learning_rate": 5e-06,
      "loss": 0.0304,
      "reward": 0.26739619076251986,
      "reward_std": 1.3646116882562638,
      "rewards/reward_func": 0.26739619076251986,
      "step": 4050,
      "toxic_reward": 3.6070310473442078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.275,
      "epoch": 0.9593572778827977,
      "format_reward": -0.25,
      "grad_norm": 1.021411657333374,
      "image_reward": 0.281744384765625,
      "kl": 2.6961711190640925,
      "learning_rate": 5e-06,
      "loss": 0.0528,
      "reward": 0.1087444543838501,
      "reward_std": 0.9241739958524704,
      "rewards/reward_func": 0.1087444543838501,
      "step": 4060,
      "toxic_reward": 4.196173495054245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.575,
      "epoch": 0.9617202268431002,
      "format_reward": -0.25,
      "grad_norm": 0.7931532859802246,
      "image_reward": 0.25716959685087204,
      "kl": 5.984370514377952,
      "learning_rate": 5e-06,
      "loss": -0.1524,
      "reward": 0.09075822830200195,
      "reward_std": 1.12701465934515,
      "rewards/reward_func": 0.09075822830200195,
      "step": 4070,
      "toxic_reward": 4.376713454723358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.375,
      "epoch": 0.9640831758034026,
      "format_reward": -0.5,
      "grad_norm": 0.7085260152816772,
      "image_reward": 0.2577000930905342,
      "kl": 1.8822400705888866,
      "learning_rate": 5e-06,
      "loss": 0.0404,
      "reward": 0.17331230640411377,
      "reward_std": 1.6633539475500583,
      "rewards/reward_func": 0.17331230640411377,
      "step": 4080,
      "toxic_reward": 4.294411969184876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.8,
      "epoch": 0.9664461247637051,
      "format_reward": 0.0,
      "grad_norm": 1.0063364505767822,
      "image_reward": 0.250091552734375,
      "kl": 0.14847910068929196,
      "learning_rate": 5e-06,
      "loss": -0.0732,
      "reward": 1.0602999448776245,
      "reward_std": 0.6203169705346226,
      "rewards/reward_func": 1.0602999448776245,
      "step": 4090,
      "toxic_reward": 4.171542119979859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.85,
      "epoch": 0.9688090737240076,
      "format_reward": -0.25,
      "grad_norm": 0.6620392203330994,
      "image_reward": 0.2424652099609375,
      "kl": 0.36983290296047927,
      "learning_rate": 5e-06,
      "loss": -0.0911,
      "reward": 0.6753372728824616,
      "reward_std": 1.2773339383304119,
      "rewards/reward_func": 0.6753372728824616,
      "step": 4100,
      "toxic_reward": 4.280330467224121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.9,
      "epoch": 0.97117202268431,
      "format_reward": 0.0,
      "grad_norm": 0.8530160188674927,
      "image_reward": 0.2524658203125,
      "kl": 4.212855443544686,
      "learning_rate": 5e-06,
      "loss": 0.0813,
      "reward": 0.6060003638267517,
      "reward_std": 0.9568195153027773,
      "rewards/reward_func": 0.6060003638267517,
      "step": 4110,
      "toxic_reward": 3.8215681195259092
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.85,
      "epoch": 0.9735349716446124,
      "format_reward": -0.5,
      "grad_norm": 1.7192955017089844,
      "image_reward": 0.28581949770450593,
      "kl": 10.378714705258608,
      "learning_rate": 5e-06,
      "loss": 0.0248,
      "reward": 0.10166561603546143,
      "reward_std": 1.791293729841709,
      "rewards/reward_func": 0.10166561603546143,
      "step": 4120,
      "toxic_reward": 3.5509902030229568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.725,
      "epoch": 0.975897920604915,
      "format_reward": 0.0,
      "grad_norm": 0.8529999852180481,
      "image_reward": 0.2569305419921875,
      "kl": 8.308781201578677,
      "learning_rate": 5e-06,
      "loss": 0.0001,
      "reward": 0.7349396765232086,
      "reward_std": 0.4486356295645237,
      "rewards/reward_func": 0.7349396765232086,
      "step": 4130,
      "toxic_reward": 4.5674937725067135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.925,
      "epoch": 0.9782608695652174,
      "format_reward": 0.0,
      "grad_norm": 0.9192355275154114,
      "image_reward": 0.273162841796875,
      "kl": 2.731711974926293,
      "learning_rate": 5e-06,
      "loss": -0.0006,
      "reward": -0.20568010210990906,
      "reward_std": 0.6350222621113062,
      "rewards/reward_func": -0.20568010210990906,
      "step": 4140,
      "toxic_reward": 4.01632958650589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.875,
      "epoch": 0.9806238185255198,
      "format_reward": 0.0,
      "grad_norm": 0.7154003977775574,
      "image_reward": 0.25846659392118454,
      "kl": 3.1543860264122485,
      "learning_rate": 5e-06,
      "loss": -0.0384,
      "reward": 0.15666076242923738,
      "reward_std": 1.1065492704510689,
      "rewards/reward_func": 0.15666076242923738,
      "step": 4150,
      "toxic_reward": 3.2047137916088104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.225,
      "epoch": 0.9829867674858223,
      "format_reward": -0.25,
      "grad_norm": 0.6323632001876831,
      "image_reward": 0.2768702179193497,
      "kl": 4.070834948495031,
      "learning_rate": 5e-06,
      "loss": 0.1427,
      "reward": 0.21166958212852477,
      "reward_std": 1.1970111442729832,
      "rewards/reward_func": 0.21166958212852477,
      "step": 4160,
      "toxic_reward": 4.128625917434692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.425,
      "epoch": 0.9853497164461248,
      "format_reward": 0.0,
      "grad_norm": 0.5803432464599609,
      "image_reward": 0.257720947265625,
      "kl": 1.2115541946142911,
      "learning_rate": 5e-06,
      "loss": -0.0251,
      "reward": 0.26483882069587705,
      "reward_std": 0.8841663489118219,
      "rewards/reward_func": 0.26483882069587705,
      "step": 4170,
      "toxic_reward": 3.953411507606506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.05,
      "epoch": 0.9877126654064272,
      "format_reward": -0.25,
      "grad_norm": 1.0321141481399536,
      "image_reward": 0.2662984222173691,
      "kl": 12.658018402941526,
      "learning_rate": 5e-06,
      "loss": 0.0031,
      "reward": 0.850147670507431,
      "reward_std": 1.0917948484420776,
      "rewards/reward_func": 0.850147670507431,
      "step": 4180,
      "toxic_reward": 4.424872517585754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.775,
      "epoch": 0.9900756143667296,
      "format_reward": -0.25,
      "grad_norm": 2.934152603149414,
      "image_reward": 0.28038330078125,
      "kl": 0.22834131643176078,
      "learning_rate": 5e-06,
      "loss": -0.12,
      "reward": -0.05527897924184799,
      "reward_std": 1.090353344194591,
      "rewards/reward_func": -0.05527897924184799,
      "step": 4190,
      "toxic_reward": 4.094451707601547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.175,
      "epoch": 0.9924385633270322,
      "format_reward": -0.5,
      "grad_norm": 1.2237070798873901,
      "image_reward": 0.28351847380399703,
      "kl": 11.847508652508258,
      "learning_rate": 5e-06,
      "loss": -0.078,
      "reward": -0.27686416208744047,
      "reward_std": 1.4433475863188505,
      "rewards/reward_func": -0.27686416208744047,
      "step": 4200,
      "toxic_reward": 4.032291853427887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.3,
      "epoch": 0.9948015122873346,
      "format_reward": 0.0,
      "grad_norm": 1.00357985496521,
      "image_reward": 0.26925506591796877,
      "kl": 6.287641528248787,
      "learning_rate": 5e-06,
      "loss": -0.0509,
      "reward": 0.26841793656349183,
      "reward_std": 0.7431968785822392,
      "rewards/reward_func": 0.26841793656349183,
      "step": 4210,
      "toxic_reward": 3.797722101211548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.4,
      "epoch": 0.997164461247637,
      "format_reward": -0.5,
      "grad_norm": 1.412477731704712,
      "image_reward": 0.2857859283685684,
      "kl": 0.20840035788714886,
      "learning_rate": 5e-06,
      "loss": -0.01,
      "reward": 0.49851550459861754,
      "reward_std": 1.4509758695960044,
      "rewards/reward_func": 0.49851550459861754,
      "step": 4220,
      "toxic_reward": 4.4275671482086185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.575,
      "epoch": 0.9995274102079396,
      "format_reward": 0.0,
      "grad_norm": 1.3644284009933472,
      "image_reward": 0.2857421875,
      "kl": 14.352099673077465,
      "learning_rate": 5e-06,
      "loss": -0.1247,
      "reward": 0.5606966435909271,
      "reward_std": 0.5899959981441498,
      "rewards/reward_func": 0.5606966435909271,
      "step": 4230,
      "toxic_reward": 4.210422110557556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.975,
      "epoch": 1.001890359168242,
      "format_reward": -0.25,
      "grad_norm": 4.210316181182861,
      "image_reward": 0.28163655698299406,
      "kl": 6.0347686521708965,
      "learning_rate": 5e-06,
      "loss": -0.0289,
      "reward": 0.11643823981285095,
      "reward_std": 1.107480544038117,
      "rewards/reward_func": 0.11643823981285095,
      "step": 4240,
      "toxic_reward": 3.994339680671692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.3,
      "epoch": 1.0042533081285445,
      "format_reward": 0.0,
      "grad_norm": 3.516270637512207,
      "image_reward": 0.2680999755859375,
      "kl": 1.3430524323135615,
      "learning_rate": 5e-06,
      "loss": 0.0258,
      "reward": 1.2558865308761598,
      "reward_std": 0.9449932537972927,
      "rewards/reward_func": 1.2558865308761598,
      "step": 4250,
      "toxic_reward": 4.3164361953735355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.1,
      "epoch": 1.0066162570888468,
      "format_reward": 0.0,
      "grad_norm": 2.1102194786071777,
      "image_reward": 0.25366058349609377,
      "kl": 0.149768141284585,
      "learning_rate": 5e-06,
      "loss": -0.0742,
      "reward": 0.06790508627891541,
      "reward_std": 0.6080379813909531,
      "rewards/reward_func": 0.06790508627891541,
      "step": 4260,
      "toxic_reward": 4.386375617980957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.325,
      "epoch": 1.0089792060491494,
      "format_reward": -0.25,
      "grad_norm": 4.862875461578369,
      "image_reward": 0.2832529693841934,
      "kl": 2.053416795656085,
      "learning_rate": 5e-06,
      "loss": -0.0907,
      "reward": 0.30691148042678834,
      "reward_std": 1.6981020882725715,
      "rewards/reward_func": 0.30691148042678834,
      "step": 4270,
      "toxic_reward": 3.5000792026519774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.05,
      "epoch": 1.011342155009452,
      "format_reward": -0.25,
      "grad_norm": 3.5172159671783447,
      "image_reward": 0.2263885498046875,
      "kl": 0.14171482473611832,
      "learning_rate": 5e-06,
      "loss": -0.0082,
      "reward": 0.5368665099143982,
      "reward_std": 1.4538173630833626,
      "rewards/reward_func": 0.5368665099143982,
      "step": 4280,
      "toxic_reward": 4.501732063293457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.1,
      "epoch": 1.0137051039697542,
      "format_reward": 0.0,
      "grad_norm": 0.6869735717773438,
      "image_reward": 0.266131591796875,
      "kl": 17.507234007120132,
      "learning_rate": 5e-06,
      "loss": -0.0165,
      "reward": 0.595378065109253,
      "reward_std": 0.6132703861221671,
      "rewards/reward_func": 0.595378065109253,
      "step": 4290,
      "toxic_reward": 4.223924076557159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.8,
      "epoch": 1.0160680529300568,
      "format_reward": -0.5,
      "grad_norm": 7.046621322631836,
      "image_reward": 0.2617146819829941,
      "kl": 0.1885729007422924,
      "learning_rate": 5e-06,
      "loss": -0.0513,
      "reward": 0.02859283685684204,
      "reward_std": 1.7149874530732632,
      "rewards/reward_func": 0.02859283685684204,
      "step": 4300,
      "toxic_reward": 4.371392369270325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.6,
      "epoch": 1.018431001890359,
      "format_reward": -0.25,
      "grad_norm": 4.381049156188965,
      "image_reward": 0.2654388427734375,
      "kl": 1.2927639432251454,
      "learning_rate": 5e-06,
      "loss": -0.0052,
      "reward": -0.0640803337097168,
      "reward_std": 1.4367546334862709,
      "rewards/reward_func": -0.0640803337097168,
      "step": 4310,
      "toxic_reward": 4.212549781799316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.9,
      "epoch": 1.0207939508506616,
      "format_reward": 0.0,
      "grad_norm": 1.0743227005004883,
      "image_reward": 0.255714924633503,
      "kl": 16.378241488710046,
      "learning_rate": 5e-06,
      "loss": -0.1016,
      "reward": 0.3336408376693726,
      "reward_std": 0.6735909695737063,
      "rewards/reward_func": 0.3336408376693726,
      "step": 4320,
      "toxic_reward": 4.5154483914375305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.4,
      "epoch": 1.0231568998109641,
      "format_reward": -0.25,
      "grad_norm": 1.6660760641098022,
      "image_reward": 0.24780476838350296,
      "kl": 11.463303370773792,
      "learning_rate": 5e-06,
      "loss": -0.0995,
      "reward": -0.0025091707706451417,
      "reward_std": 1.6661910176277162,
      "rewards/reward_func": -0.0025091707706451417,
      "step": 4330,
      "toxic_reward": 3.852264070510864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.4,
      "epoch": 1.0255198487712665,
      "format_reward": 0.0,
      "grad_norm": 4.4377121925354,
      "image_reward": 0.2486114501953125,
      "kl": 0.12634929567575454,
      "learning_rate": 5e-06,
      "loss": 0.0148,
      "reward": 0.518048095703125,
      "reward_std": 1.0762871712446214,
      "rewards/reward_func": 0.518048095703125,
      "step": 4340,
      "toxic_reward": 4.010181951522827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.875,
      "epoch": 1.027882797731569,
      "format_reward": -0.25,
      "grad_norm": 1.9050147533416748,
      "image_reward": 0.2529998779296875,
      "kl": 0.20360449738800526,
      "learning_rate": 5e-06,
      "loss": 0.0104,
      "reward": -0.05591415464878082,
      "reward_std": 1.2363329231739044,
      "rewards/reward_func": -0.05591415464878082,
      "step": 4350,
      "toxic_reward": 4.434300184249878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.9,
      "epoch": 1.0302457466918715,
      "format_reward": -0.25,
      "grad_norm": 2.783447742462158,
      "image_reward": 0.25158691257238386,
      "kl": 0.14820914287120104,
      "learning_rate": 5e-06,
      "loss": 0.0709,
      "reward": 0.2038910448551178,
      "reward_std": 1.113127877563238,
      "rewards/reward_func": 0.2038910448551178,
      "step": 4360,
      "toxic_reward": 4.226523244380951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.85,
      "epoch": 1.0326086956521738,
      "format_reward": -0.25,
      "grad_norm": 0.6769667863845825,
      "image_reward": 0.2698781341314316,
      "kl": 1.560221792012453,
      "learning_rate": 5e-06,
      "loss": 0.0453,
      "reward": -0.25547429323196413,
      "reward_std": 1.2353890612721443,
      "rewards/reward_func": -0.25547429323196413,
      "step": 4370,
      "toxic_reward": 4.451306319236755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.325,
      "epoch": 1.0349716446124764,
      "format_reward": -0.5,
      "grad_norm": 1.5327138900756836,
      "image_reward": 0.25219675749540327,
      "kl": 8.099627137556672,
      "learning_rate": 5e-06,
      "loss": -0.0101,
      "reward": -0.2395196735858917,
      "reward_std": 1.6747881084680558,
      "rewards/reward_func": -0.2395196735858917,
      "step": 4380,
      "toxic_reward": 4.392533135414124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.875,
      "epoch": 1.037334593572779,
      "format_reward": 0.0,
      "grad_norm": 1.2079089879989624,
      "image_reward": 0.2677134186029434,
      "kl": 20.563808789849283,
      "learning_rate": 5e-06,
      "loss": 0.0416,
      "reward": 0.5019214197993278,
      "reward_std": 1.3155438639223576,
      "rewards/reward_func": 0.5019214197993278,
      "step": 4390,
      "toxic_reward": 3.744106537103653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.575,
      "epoch": 1.0396975425330812,
      "format_reward": 0.0,
      "grad_norm": 12.300169944763184,
      "image_reward": 0.266143798828125,
      "kl": 0.33298523649573325,
      "learning_rate": 5e-06,
      "loss": 0.0325,
      "reward": 0.3428509056568146,
      "reward_std": 0.6832939319312572,
      "rewards/reward_func": 0.3428509056568146,
      "step": 4400,
      "toxic_reward": 3.938971757888794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.575,
      "epoch": 1.0420604914933838,
      "format_reward": 0.0,
      "grad_norm": 13.788394927978516,
      "image_reward": 0.2602081298828125,
      "kl": 0.14277449063956738,
      "learning_rate": 5e-06,
      "loss": 0.0799,
      "reward": 0.5224148035049438,
      "reward_std": 0.5329875692725181,
      "rewards/reward_func": 0.5224148035049438,
      "step": 4410,
      "toxic_reward": 4.6161150455474855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.45,
      "epoch": 1.0444234404536863,
      "format_reward": 0.0,
      "grad_norm": 9.009355545043945,
      "image_reward": 0.2745361328125,
      "kl": 0.1908944919705391,
      "learning_rate": 5e-06,
      "loss": 0.0428,
      "reward": 0.5507814303040505,
      "reward_std": 0.7364906007423997,
      "rewards/reward_func": 0.5507814303040505,
      "step": 4420,
      "toxic_reward": 3.738240921497345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.375,
      "epoch": 1.0467863894139886,
      "format_reward": -0.25,
      "grad_norm": 0.9596161842346191,
      "image_reward": 0.2600412994623184,
      "kl": 0.2493920259177685,
      "learning_rate": 5e-06,
      "loss": 0.0546,
      "reward": 0.3896596789360046,
      "reward_std": 1.1463438659906386,
      "rewards/reward_func": 0.3896596789360046,
      "step": 4430,
      "toxic_reward": 4.299828362464905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.6,
      "epoch": 1.0491493383742911,
      "format_reward": -0.5,
      "grad_norm": 16.468900680541992,
      "image_reward": 0.2660593673586845,
      "kl": 1.7080695651471616,
      "learning_rate": 5e-06,
      "loss": 0.0381,
      "reward": 0.23900684118270873,
      "reward_std": 1.4474023096263409,
      "rewards/reward_func": 0.23900684118270873,
      "step": 4440,
      "toxic_reward": 4.323360848426819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.175,
      "epoch": 1.0515122873345937,
      "format_reward": 0.0,
      "grad_norm": 9.211010932922363,
      "image_reward": 0.23970438539981842,
      "kl": 2.5637484416365623,
      "learning_rate": 5e-06,
      "loss": 0.1362,
      "reward": 1.0162243604660035,
      "reward_std": 0.6415727452374995,
      "rewards/reward_func": 1.0162243604660035,
      "step": 4450,
      "toxic_reward": 4.387140679359436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.875,
      "epoch": 1.053875236294896,
      "format_reward": -0.25,
      "grad_norm": 7.96478796005249,
      "image_reward": 0.279913330078125,
      "kl": 1.5326927796006202,
      "learning_rate": 5e-06,
      "loss": 0.135,
      "reward": -0.0030475854873657227,
      "reward_std": 1.1649701196700335,
      "rewards/reward_func": -0.0030475854873657227,
      "step": 4460,
      "toxic_reward": 4.197524422407151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.475,
      "epoch": 1.0562381852551985,
      "format_reward": 0.0,
      "grad_norm": 18.094940185546875,
      "image_reward": 0.25467529296875,
      "kl": 0.3941259577870369,
      "learning_rate": 5e-06,
      "loss": -0.0321,
      "reward": 0.24867143034934996,
      "reward_std": 0.6323847549967467,
      "rewards/reward_func": 0.24867143034934996,
      "step": 4470,
      "toxic_reward": 4.357023143768311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.6,
      "epoch": 1.0586011342155008,
      "format_reward": 0.0,
      "grad_norm": 6.004316329956055,
      "image_reward": 0.249798583984375,
      "kl": 0.6100661933422089,
      "learning_rate": 5e-06,
      "loss": 0.054,
      "reward": 0.3194288432598114,
      "reward_std": 1.00972272567451,
      "rewards/reward_func": 0.3194288432598114,
      "step": 4480,
      "toxic_reward": 3.6232224822044374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.275,
      "epoch": 1.0609640831758034,
      "format_reward": 0.0,
      "grad_norm": 16.553150177001953,
      "image_reward": 0.2777862548828125,
      "kl": 1.2566918075084685,
      "learning_rate": 5e-06,
      "loss": 0.0033,
      "reward": 0.3464995056390762,
      "reward_std": 1.0610926449298859,
      "rewards/reward_func": 0.3464995056390762,
      "step": 4490,
      "toxic_reward": 3.4315811157226563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.5,
      "epoch": 1.063327032136106,
      "format_reward": 0.0,
      "grad_norm": 21.91239356994629,
      "image_reward": 0.26402740478515624,
      "kl": 2.7657025068998338,
      "learning_rate": 5e-06,
      "loss": -0.0088,
      "reward": -0.017888635396957397,
      "reward_std": 0.36575160175561905,
      "rewards/reward_func": -0.017888635396957397,
      "step": 4500,
      "toxic_reward": 4.492252993583679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.175,
      "epoch": 1.0656899810964082,
      "format_reward": 0.0,
      "grad_norm": 1.70862877368927,
      "image_reward": 0.2537200927734375,
      "kl": 0.9243695795536041,
      "learning_rate": 5e-06,
      "loss": -0.148,
      "reward": 0.2666252374649048,
      "reward_std": 0.8290498301386833,
      "rewards/reward_func": 0.2666252374649048,
      "step": 4510,
      "toxic_reward": 4.025203084945678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.625,
      "epoch": 1.0680529300567108,
      "format_reward": 0.0,
      "grad_norm": 4.76298189163208,
      "image_reward": 0.2544342041015625,
      "kl": 1.2314461708068847,
      "learning_rate": 5e-06,
      "loss": 0.0379,
      "reward": 0.1297641634941101,
      "reward_std": 0.7050925550982357,
      "rewards/reward_func": 0.1297641634941101,
      "step": 4520,
      "toxic_reward": 4.568165302276611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.825,
      "epoch": 1.0704158790170133,
      "format_reward": 0.0,
      "grad_norm": 8.698065757751465,
      "image_reward": 0.274078369140625,
      "kl": 1.325176051259041,
      "learning_rate": 5e-06,
      "loss": -0.0398,
      "reward": 0.10026351213455201,
      "reward_std": 0.812692479044199,
      "rewards/reward_func": 0.10026351213455201,
      "step": 4530,
      "toxic_reward": 3.9686192631721497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.3,
      "epoch": 1.0727788279773156,
      "format_reward": 0.0,
      "grad_norm": 19.877777099609375,
      "image_reward": 0.2792388916015625,
      "kl": 0.9205800026655198,
      "learning_rate": 5e-06,
      "loss": 0.1066,
      "reward": 0.5690743923187256,
      "reward_std": 0.6653784658759833,
      "rewards/reward_func": 0.5690743923187256,
      "step": 4540,
      "toxic_reward": 3.964191234111786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.525,
      "epoch": 1.0751417769376181,
      "format_reward": -0.25,
      "grad_norm": 2.515148878097534,
      "image_reward": 0.23823343813419343,
      "kl": 2.8265303134918214,
      "learning_rate": 5e-06,
      "loss": -0.1035,
      "reward": 0.6135944664478302,
      "reward_std": 1.5456651039421558,
      "rewards/reward_func": 0.6135944664478302,
      "step": 4550,
      "toxic_reward": 4.405286359786987
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.85,
      "epoch": 1.0775047258979207,
      "format_reward": 0.0,
      "grad_norm": 16.40328025817871,
      "image_reward": 0.26103515625,
      "kl": 2.2788069248199463,
      "learning_rate": 5e-06,
      "loss": 0.0091,
      "reward": 0.10576534271240234,
      "reward_std": 0.39959471523761747,
      "rewards/reward_func": 0.10576534271240234,
      "step": 4560,
      "toxic_reward": 4.633650445938111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.7,
      "epoch": 1.079867674858223,
      "format_reward": -0.5,
      "grad_norm": 7.5780229568481445,
      "image_reward": 0.26860554963350297,
      "kl": 4.05745484828949,
      "learning_rate": 5e-06,
      "loss": 0.0631,
      "reward": 0.6903072118759155,
      "reward_std": 1.8377123966813087,
      "rewards/reward_func": 0.6903072118759155,
      "step": 4570,
      "toxic_reward": 4.404474878311158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.25,
      "epoch": 1.0822306238185255,
      "format_reward": -0.25,
      "grad_norm": 2.605886936187744,
      "image_reward": 0.26128031462430956,
      "kl": 1.1529910147190094,
      "learning_rate": 5e-06,
      "loss": 0.0061,
      "reward": -0.4064223051071167,
      "reward_std": 1.0936089092865586,
      "rewards/reward_func": -0.4064223051071167,
      "step": 4580,
      "toxic_reward": 4.17680971622467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.225,
      "epoch": 1.084593572778828,
      "format_reward": 0.0,
      "grad_norm": 2.201918601989746,
      "image_reward": 0.26739501953125,
      "kl": 0.7634936004877091,
      "learning_rate": 5e-06,
      "loss": 0.0832,
      "reward": 0.2786406099796295,
      "reward_std": 0.7699430305510759,
      "rewards/reward_func": 0.2786406099796295,
      "step": 4590,
      "toxic_reward": 4.203878152370453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.75,
      "epoch": 1.0869565217391304,
      "format_reward": 0.0,
      "grad_norm": 4.329306125640869,
      "image_reward": 0.2691864013671875,
      "kl": 1.415566897392273,
      "learning_rate": 5e-06,
      "loss": 0.0091,
      "reward": 0.031065577268600465,
      "reward_std": 0.9241972327232361,
      "rewards/reward_func": 0.031065577268600465,
      "step": 4600,
      "toxic_reward": 3.4664461970329286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.825,
      "epoch": 1.089319470699433,
      "format_reward": 0.0,
      "grad_norm": 15.773272514343262,
      "image_reward": 0.23636678010225295,
      "kl": 3.1093257188797,
      "learning_rate": 5e-06,
      "loss": -0.0048,
      "reward": 0.6077887773513794,
      "reward_std": 0.9942519944161177,
      "rewards/reward_func": 0.6077887773513794,
      "step": 4610,
      "toxic_reward": 4.182659006118774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.6,
      "epoch": 1.0916824196597354,
      "format_reward": -0.25,
      "grad_norm": 4.842249870300293,
      "image_reward": 0.2884033203125,
      "kl": 0.8512112647294998,
      "learning_rate": 5e-06,
      "loss": 0.0029,
      "reward": 0.2840136528015137,
      "reward_std": 1.34358575232327,
      "rewards/reward_func": 0.2840136528015137,
      "step": 4620,
      "toxic_reward": 4.202986550331116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.0,
      "epoch": 1.0940453686200378,
      "format_reward": 0.0,
      "grad_norm": 12.111598014831543,
      "image_reward": 0.26724853515625,
      "kl": 0.5160227678716183,
      "learning_rate": 5e-06,
      "loss": -0.1139,
      "reward": 0.41757542341947557,
      "reward_std": 0.6193137221038342,
      "rewards/reward_func": 0.41757542341947557,
      "step": 4630,
      "toxic_reward": 3.6429463922977448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.475,
      "epoch": 1.0964083175803403,
      "format_reward": 0.0,
      "grad_norm": 5.654786586761475,
      "image_reward": 0.2576507568359375,
      "kl": 0.6431491911411286,
      "learning_rate": 5e-06,
      "loss": -0.0209,
      "reward": 0.4444656491279602,
      "reward_std": 0.8271868824958801,
      "rewards/reward_func": 0.4444656491279602,
      "step": 4640,
      "toxic_reward": 3.751771080493927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.225,
      "epoch": 1.0987712665406426,
      "format_reward": 0.0,
      "grad_norm": 3.850701332092285,
      "image_reward": 0.25439249724149704,
      "kl": 3.444407519698143,
      "learning_rate": 5e-06,
      "loss": 0.0854,
      "reward": 0.643358188867569,
      "reward_std": 0.8931491523981094,
      "rewards/reward_func": 0.643358188867569,
      "step": 4650,
      "toxic_reward": 4.321841323375702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.575,
      "epoch": 1.1011342155009451,
      "format_reward": 0.0,
      "grad_norm": 6.3553853034973145,
      "image_reward": 0.2528656005859375,
      "kl": 0.5195316299796104,
      "learning_rate": 5e-06,
      "loss": -0.0275,
      "reward": 0.4554763913154602,
      "reward_std": 0.8332011103630066,
      "rewards/reward_func": 0.4554763913154602,
      "step": 4660,
      "toxic_reward": 4.327652913331986
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.925,
      "epoch": 1.1034971644612477,
      "format_reward": 0.0,
      "grad_norm": 7.020429611206055,
      "image_reward": 0.24530232697725296,
      "kl": 1.074008372426033,
      "learning_rate": 5e-06,
      "loss": 0.1332,
      "reward": 0.08963438272476196,
      "reward_std": 1.1591505765914918,
      "rewards/reward_func": 0.08963438272476196,
      "step": 4670,
      "toxic_reward": 3.8221271514892576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.675,
      "epoch": 1.10586011342155,
      "format_reward": 0.0,
      "grad_norm": 1.8656316995620728,
      "image_reward": 0.23163909912109376,
      "kl": 1.6141413852572442,
      "learning_rate": 5e-06,
      "loss": 0.0496,
      "reward": 0.4311521232128143,
      "reward_std": 0.39210873320698736,
      "rewards/reward_func": 0.4311521232128143,
      "step": 4680,
      "toxic_reward": 4.517966604232788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.2,
      "epoch": 1.1082230623818525,
      "format_reward": 0.0,
      "grad_norm": 13.881125450134277,
      "image_reward": 0.2380462646484375,
      "kl": 0.6565065160393715,
      "learning_rate": 5e-06,
      "loss": -0.0278,
      "reward": 0.3582367777824402,
      "reward_std": 1.0096068516373635,
      "rewards/reward_func": 0.3582367777824402,
      "step": 4690,
      "toxic_reward": 3.9490260004997255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.2,
      "epoch": 1.110586011342155,
      "format_reward": -0.25,
      "grad_norm": 6.552460193634033,
      "image_reward": 0.24650166779756547,
      "kl": 2.4836434960365295,
      "learning_rate": 5e-06,
      "loss": -0.055,
      "reward": 0.850802743434906,
      "reward_std": 1.5017553605139256,
      "rewards/reward_func": 0.850802743434906,
      "step": 4700,
      "toxic_reward": 4.000320458412171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.35,
      "epoch": 1.1129489603024574,
      "format_reward": 0.0,
      "grad_norm": 10.16213607788086,
      "image_reward": 0.2479156494140625,
      "kl": 9.667583072185517,
      "learning_rate": 5e-06,
      "loss": 0.0035,
      "reward": 0.7445069432258606,
      "reward_std": 0.7123569492250681,
      "rewards/reward_func": 0.7445069432258606,
      "step": 4710,
      "toxic_reward": 4.515477871894836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.825,
      "epoch": 1.11531190926276,
      "format_reward": 0.0,
      "grad_norm": 11.000924110412598,
      "image_reward": 0.2601796478033066,
      "kl": 1.5108904749155045,
      "learning_rate": 5e-06,
      "loss": 0.0823,
      "reward": 0.5056971669197082,
      "reward_std": 0.6825690733268857,
      "rewards/reward_func": 0.5056971669197082,
      "step": 4720,
      "toxic_reward": 4.039038109779358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.9,
      "epoch": 1.1176748582230625,
      "format_reward": -0.25,
      "grad_norm": 10.222740173339844,
      "image_reward": 0.25692138671875,
      "kl": 0.35422504395246507,
      "learning_rate": 5e-06,
      "loss": 0.0093,
      "reward": 0.08535944372415542,
      "reward_std": 1.423003512620926,
      "rewards/reward_func": 0.08535944372415542,
      "step": 4730,
      "toxic_reward": 3.4993788480758665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.55,
      "epoch": 1.1200378071833648,
      "format_reward": -0.25,
      "grad_norm": 2.3486738204956055,
      "image_reward": 0.24800923615694045,
      "kl": 0.38842023983597757,
      "learning_rate": 5e-06,
      "loss": -0.0107,
      "reward": 0.1211450919508934,
      "reward_std": 1.3706756496801973,
      "rewards/reward_func": 0.1211450919508934,
      "step": 4740,
      "toxic_reward": 3.2782628774642943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.325,
      "epoch": 1.1224007561436673,
      "format_reward": 0.0,
      "grad_norm": 4.218822479248047,
      "image_reward": 0.251934814453125,
      "kl": 1.4006462961435318,
      "learning_rate": 5e-06,
      "loss": 0.0701,
      "reward": 0.9292663365602494,
      "reward_std": 0.874046965315938,
      "rewards/reward_func": 0.9292663365602494,
      "step": 4750,
      "toxic_reward": 4.4716246843338014
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.45,
      "epoch": 1.1247637051039698,
      "format_reward": 0.0,
      "grad_norm": 13.548481941223145,
      "image_reward": 0.276861572265625,
      "kl": 0.5785035833716392,
      "learning_rate": 5e-06,
      "loss": -0.0177,
      "reward": 0.1986662968993187,
      "reward_std": 0.7839731447398662,
      "rewards/reward_func": 0.1986662968993187,
      "step": 4760,
      "toxic_reward": 4.128668719530106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.975,
      "epoch": 1.1271266540642721,
      "format_reward": 0.0,
      "grad_norm": 10.347504615783691,
      "image_reward": 0.2632904052734375,
      "kl": 0.28924584165215494,
      "learning_rate": 5e-06,
      "loss": -0.0536,
      "reward": 0.40365022569894793,
      "reward_std": 0.6283778937533497,
      "rewards/reward_func": 0.40365022569894793,
      "step": 4770,
      "toxic_reward": 3.78736280053854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.775,
      "epoch": 1.1294896030245747,
      "format_reward": -0.75,
      "grad_norm": 29.38702964782715,
      "image_reward": 0.27183634638786314,
      "kl": 8.101900951564312,
      "learning_rate": 5e-06,
      "loss": -0.0473,
      "reward": -0.11305050253868103,
      "reward_std": 2.1815814077854156,
      "rewards/reward_func": -0.11305050253868103,
      "step": 4780,
      "toxic_reward": 3.949468755722046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.7,
      "epoch": 1.1318525519848772,
      "format_reward": -0.75,
      "grad_norm": 8.19861125946045,
      "image_reward": 0.2731597885489464,
      "kl": 5.514032608270645,
      "learning_rate": 5e-06,
      "loss": 0.0809,
      "reward": -0.5878833532333374,
      "reward_std": 1.700104326196015,
      "rewards/reward_func": -0.5878833532333374,
      "step": 4790,
      "toxic_reward": 4.362279486656189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.575,
      "epoch": 1.1342155009451795,
      "format_reward": 0.0,
      "grad_norm": 25.879568099975586,
      "image_reward": 0.272625732421875,
      "kl": 0.41564694195985796,
      "learning_rate": 5e-06,
      "loss": 0.0466,
      "reward": 0.5246647775173188,
      "reward_std": 0.5603986160829664,
      "rewards/reward_func": 0.5246647775173188,
      "step": 4800,
      "toxic_reward": 4.4845054864883425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.175,
      "epoch": 1.136578449905482,
      "format_reward": -0.25,
      "grad_norm": 6.490880966186523,
      "image_reward": 0.27078043669462204,
      "kl": 0.39700448513031006,
      "learning_rate": 5e-06,
      "loss": 0.0442,
      "reward": 0.10371096134185791,
      "reward_std": 1.3051490228623153,
      "rewards/reward_func": 0.10371096134185791,
      "step": 4810,
      "toxic_reward": 4.362593126296997
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.025,
      "epoch": 1.1389413988657844,
      "format_reward": 0.0,
      "grad_norm": 10.680285453796387,
      "image_reward": 0.2447296142578125,
      "kl": 0.43964013159275056,
      "learning_rate": 5e-06,
      "loss": -0.1088,
      "reward": 0.43211621046066284,
      "reward_std": 0.5677682287991047,
      "rewards/reward_func": 0.43211621046066284,
      "step": 4820,
      "toxic_reward": 4.520205068588257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.225,
      "epoch": 1.141304347826087,
      "format_reward": -0.5,
      "grad_norm": 7.846988201141357,
      "image_reward": 0.24936320036649703,
      "kl": 0.3737114042043686,
      "learning_rate": 5e-06,
      "loss": 0.1324,
      "reward": -0.08488219976425171,
      "reward_std": 1.6377468653023244,
      "rewards/reward_func": -0.08488219976425171,
      "step": 4830,
      "toxic_reward": 3.979211616516113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.95,
      "epoch": 1.1436672967863895,
      "format_reward": -0.25,
      "grad_norm": 13.332221031188965,
      "image_reward": 0.27100830078125,
      "kl": 0.39419813454151154,
      "learning_rate": 5e-06,
      "loss": -0.0752,
      "reward": 0.9029350757598877,
      "reward_std": 1.455178501456976,
      "rewards/reward_func": 0.9029350757598877,
      "step": 4840,
      "toxic_reward": 3.6434609413146974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.125,
      "epoch": 1.146030245746692,
      "format_reward": 0.0,
      "grad_norm": 3.2025651931762695,
      "image_reward": 0.281640625,
      "kl": 7.567617936432361,
      "learning_rate": 5e-06,
      "loss": -0.0926,
      "reward": 0.7164658069610595,
      "reward_std": 0.6624833345413208,
      "rewards/reward_func": 0.7164658069610595,
      "step": 4850,
      "toxic_reward": 3.8413574934005736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.975,
      "epoch": 1.1483931947069943,
      "format_reward": 0.0,
      "grad_norm": 9.695226669311523,
      "image_reward": 0.24990997314453126,
      "kl": 0.28165399581193923,
      "learning_rate": 5e-06,
      "loss": 0.0188,
      "reward": -0.12237508296966552,
      "reward_std": 0.6198875203728675,
      "rewards/reward_func": -0.12237508296966552,
      "step": 4860,
      "toxic_reward": 4.485203766822815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.45,
      "epoch": 1.1507561436672968,
      "format_reward": 0.0,
      "grad_norm": 2.5677099227905273,
      "image_reward": 0.2783660888671875,
      "kl": 0.29035804942250254,
      "learning_rate": 5e-06,
      "loss": -0.0618,
      "reward": 0.050651901960372926,
      "reward_std": 1.2044988840818405,
      "rewards/reward_func": 0.050651901960372926,
      "step": 4870,
      "toxic_reward": 3.6688124537467957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.9,
      "epoch": 1.1531190926275992,
      "format_reward": 0.0,
      "grad_norm": 4.83213996887207,
      "image_reward": 0.2756866455078125,
      "kl": 5.533606587722898,
      "learning_rate": 5e-06,
      "loss": -0.1798,
      "reward": 0.5692965686321259,
      "reward_std": 1.0450827227905393,
      "rewards/reward_func": 0.5692965686321259,
      "step": 4880,
      "toxic_reward": 3.439787745475769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.375,
      "epoch": 1.1554820415879017,
      "format_reward": 0.0,
      "grad_norm": 1.0500257015228271,
      "image_reward": 0.258331298828125,
      "kl": 0.11663263067603111,
      "learning_rate": 5e-06,
      "loss": 0.0311,
      "reward": 0.4637997090816498,
      "reward_std": 0.9648044936358928,
      "rewards/reward_func": 0.4637997090816498,
      "step": 4890,
      "toxic_reward": 4.382505106925964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.5,
      "epoch": 1.1578449905482042,
      "format_reward": 0.0,
      "grad_norm": 7.189860820770264,
      "image_reward": 0.2593902587890625,
      "kl": 0.15962190218269826,
      "learning_rate": 5e-06,
      "loss": 0.0056,
      "reward": 0.5262487173080445,
      "reward_std": 1.052651860564947,
      "rewards/reward_func": 0.5262487173080445,
      "step": 4900,
      "toxic_reward": 4.166365385055542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.575,
      "epoch": 1.1602079395085065,
      "format_reward": 0.0,
      "grad_norm": 1.5136041641235352,
      "image_reward": 0.277166748046875,
      "kl": 0.14849806036800145,
      "learning_rate": 5e-06,
      "loss": -0.0034,
      "reward": 0.22453336119651796,
      "reward_std": 0.5165121786296367,
      "rewards/reward_func": 0.22453336119651796,
      "step": 4910,
      "toxic_reward": 3.900139307975769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.55,
      "epoch": 1.162570888468809,
      "format_reward": 0.0,
      "grad_norm": 0.6905107498168945,
      "image_reward": 0.26757049560546875,
      "kl": 0.17389641776680947,
      "learning_rate": 5e-06,
      "loss": 0.0273,
      "reward": 0.2769235372543335,
      "reward_std": 0.8026977114379406,
      "rewards/reward_func": 0.2769235372543335,
      "step": 4920,
      "toxic_reward": 4.421657228469849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.9,
      "epoch": 1.1649338374291116,
      "format_reward": 0.0,
      "grad_norm": 0.7038688063621521,
      "image_reward": 0.23498077392578126,
      "kl": 0.1468098048120737,
      "learning_rate": 5e-06,
      "loss": -0.0593,
      "reward": 0.6022326171398162,
      "reward_std": 0.8370201224461198,
      "rewards/reward_func": 0.6022326171398162,
      "step": 4930,
      "toxic_reward": 4.272796273231506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.575,
      "epoch": 1.167296786389414,
      "format_reward": 0.0,
      "grad_norm": 3.3997626304626465,
      "image_reward": 0.2218317672610283,
      "kl": 10.733999550715088,
      "learning_rate": 5e-06,
      "loss": -0.0239,
      "reward": 0.5406073331832886,
      "reward_std": 1.1294488459825516,
      "rewards/reward_func": 0.5406073331832886,
      "step": 4940,
      "toxic_reward": 4.010496520996094
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.75,
      "epoch": 1.1696597353497165,
      "format_reward": 0.0,
      "grad_norm": 13.437244415283203,
      "image_reward": 0.260540771484375,
      "kl": 0.4533839326351881,
      "learning_rate": 5e-06,
      "loss": 0.0308,
      "reward": 0.6349693357944488,
      "reward_std": 0.9300125196576119,
      "rewards/reward_func": 0.6349693357944488,
      "step": 4950,
      "toxic_reward": 3.9825836658477782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.275,
      "epoch": 1.172022684310019,
      "format_reward": 0.0,
      "grad_norm": 5.643482208251953,
      "image_reward": 0.25181121826171876,
      "kl": 0.7526591405272484,
      "learning_rate": 5e-06,
      "loss": 0.0702,
      "reward": 0.2168402910232544,
      "reward_std": 0.8874317653477192,
      "rewards/reward_func": 0.2168402910232544,
      "step": 4960,
      "toxic_reward": 4.36525526046753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.125,
      "epoch": 1.1743856332703213,
      "format_reward": -0.25,
      "grad_norm": 2.1135120391845703,
      "image_reward": 0.27721354067325593,
      "kl": 1.7936469875276089,
      "learning_rate": 5e-06,
      "loss": 0.0616,
      "reward": -0.168658310174942,
      "reward_std": 1.076946148276329,
      "rewards/reward_func": -0.168658310174942,
      "step": 4970,
      "toxic_reward": 4.423034191131592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.2,
      "epoch": 1.1767485822306238,
      "format_reward": 0.0,
      "grad_norm": 1.0600641965866089,
      "image_reward": 0.22789459228515624,
      "kl": 3.3630725659430025,
      "learning_rate": 5e-06,
      "loss": 0.0228,
      "reward": 0.7056062936782836,
      "reward_std": 0.9683291807770729,
      "rewards/reward_func": 0.7056062936782836,
      "step": 4980,
      "toxic_reward": 4.235305881500244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.025,
      "epoch": 1.1791115311909262,
      "format_reward": -0.25,
      "grad_norm": 1.4251501560211182,
      "image_reward": 0.25722147673368456,
      "kl": 0.2127727370709181,
      "learning_rate": 5e-06,
      "loss": -0.0369,
      "reward": 0.7460228025913238,
      "reward_std": 1.3902123406529427,
      "rewards/reward_func": 0.7460228025913238,
      "step": 4990,
      "toxic_reward": 4.187235593795776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.85,
      "epoch": 1.1814744801512287,
      "format_reward": 0.0,
      "grad_norm": 0.9059237837791443,
      "image_reward": 0.275982666015625,
      "kl": 0.1094449780881405,
      "learning_rate": 5e-06,
      "loss": 0.0122,
      "reward": -0.013343071937561036,
      "reward_std": 0.8927877993322909,
      "rewards/reward_func": -0.013343071937561036,
      "step": 5000,
      "toxic_reward": 4.172649383544922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.475,
      "epoch": 1.1838374291115312,
      "format_reward": -0.25,
      "grad_norm": 0.676426887512207,
      "image_reward": 0.25437113344669343,
      "kl": 0.306893527135253,
      "learning_rate": 5e-06,
      "loss": 0.0597,
      "reward": 0.5460815012454987,
      "reward_std": 1.3148551121354104,
      "rewards/reward_func": 0.5460815012454987,
      "step": 5010,
      "toxic_reward": 4.169591236114502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.5,
      "epoch": 1.1862003780718338,
      "format_reward": -0.25,
      "grad_norm": 1.0359044075012207,
      "image_reward": 0.25831960141658783,
      "kl": 0.11788953803479671,
      "learning_rate": 5e-06,
      "loss": 0.0199,
      "reward": 0.008247452974319457,
      "reward_std": 1.6603192906826734,
      "rewards/reward_func": 0.008247452974319457,
      "step": 5020,
      "toxic_reward": 4.008079314231873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.7,
      "epoch": 1.188563327032136,
      "format_reward": -0.25,
      "grad_norm": 4.407492637634277,
      "image_reward": 0.25005086213350297,
      "kl": 0.16296980381011963,
      "learning_rate": 5e-06,
      "loss": 0.013,
      "reward": 0.45398043394088744,
      "reward_std": 1.4666540574282407,
      "rewards/reward_func": 0.45398043394088744,
      "step": 5030,
      "toxic_reward": 3.9480291843414306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.05,
      "epoch": 1.1909262759924386,
      "format_reward": -0.25,
      "grad_norm": 1.3405718803405762,
      "image_reward": 0.26083475798368455,
      "kl": 1.2570629265159368,
      "learning_rate": 5e-06,
      "loss": -0.0608,
      "reward": 0.24062097072601318,
      "reward_std": 1.1102397807873785,
      "rewards/reward_func": 0.24062097072601318,
      "step": 5040,
      "toxic_reward": 4.282037019729614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.475,
      "epoch": 1.193289224952741,
      "format_reward": -0.25,
      "grad_norm": 1.0421810150146484,
      "image_reward": 0.2481842041015625,
      "kl": 0.4886137153953314,
      "learning_rate": 5e-06,
      "loss": 0.0154,
      "reward": 0.35102577805519103,
      "reward_std": 1.4176109634339809,
      "rewards/reward_func": 0.35102577805519103,
      "step": 5050,
      "toxic_reward": 4.56660737991333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.775,
      "epoch": 1.1956521739130435,
      "format_reward": 0.0,
      "grad_norm": 2.4514474868774414,
      "image_reward": 0.2724589020013809,
      "kl": 17.261842382885515,
      "learning_rate": 5e-06,
      "loss": 0.0799,
      "reward": 0.2634397208690643,
      "reward_std": 0.6655941482633352,
      "rewards/reward_func": 0.2634397208690643,
      "step": 5060,
      "toxic_reward": 4.50599045753479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.675,
      "epoch": 1.198015122873346,
      "format_reward": -0.25,
      "grad_norm": 4.386458396911621,
      "image_reward": 0.2706837967038155,
      "kl": 0.41296282410621643,
      "learning_rate": 5e-06,
      "loss": -0.0407,
      "reward": 0.3763133823871613,
      "reward_std": 1.3990098256617785,
      "rewards/reward_func": 0.3763133823871613,
      "step": 5070,
      "toxic_reward": 3.8180208444595336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.55,
      "epoch": 1.2003780718336483,
      "format_reward": 0.0,
      "grad_norm": 3.708019495010376,
      "image_reward": 0.2523040771484375,
      "kl": 0.13607071787118913,
      "learning_rate": 5e-06,
      "loss": -0.0338,
      "reward": 0.09913046360015869,
      "reward_std": 0.64256557286717,
      "rewards/reward_func": 0.09913046360015869,
      "step": 5080,
      "toxic_reward": 4.473843407630921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.85,
      "epoch": 1.2027410207939508,
      "format_reward": 0.0,
      "grad_norm": 0.9288604855537415,
      "image_reward": 0.24779205322265624,
      "kl": 0.20878240577876567,
      "learning_rate": 5e-06,
      "loss": 0.041,
      "reward": 0.5819396436214447,
      "reward_std": 0.7615427184849978,
      "rewards/reward_func": 0.5819396436214447,
      "step": 5090,
      "toxic_reward": 4.673109149932861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.825,
      "epoch": 1.2051039697542534,
      "format_reward": -0.75,
      "grad_norm": 1.3514373302459717,
      "image_reward": 0.24433186948299407,
      "kl": 0.2980830356478691,
      "learning_rate": 5e-06,
      "loss": -0.0794,
      "reward": -0.4275161147117615,
      "reward_std": 2.30497971996665,
      "rewards/reward_func": -0.4275161147117615,
      "step": 5100,
      "toxic_reward": 4.221112084388733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 34.775,
      "epoch": 1.2074669187145557,
      "format_reward": 0.0,
      "grad_norm": 1.3809269666671753,
      "image_reward": 0.2739410400390625,
      "kl": 4.718816532939672,
      "learning_rate": 5e-06,
      "loss": -0.1045,
      "reward": 0.23022666573524475,
      "reward_std": 0.9735932052135468,
      "rewards/reward_func": 0.23022666573524475,
      "step": 5110,
      "toxic_reward": 3.8542242765426638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.125,
      "epoch": 1.2098298676748582,
      "format_reward": -0.75,
      "grad_norm": 1.045753836631775,
      "image_reward": 0.243878173828125,
      "kl": 0.15604666136205197,
      "learning_rate": 5e-06,
      "loss": 0.0351,
      "reward": -0.7461395561695099,
      "reward_std": 2.103620085120201,
      "rewards/reward_func": -0.7461395561695099,
      "step": 5120,
      "toxic_reward": 4.052614498138428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.3,
      "epoch": 1.2121928166351608,
      "format_reward": 0.0,
      "grad_norm": 2.4709815979003906,
      "image_reward": 0.25801239013671873,
      "kl": 0.1505513045936823,
      "learning_rate": 5e-06,
      "loss": -0.073,
      "reward": 0.789186455309391,
      "reward_std": 1.0413845662027597,
      "rewards/reward_func": 0.789186455309391,
      "step": 5130,
      "toxic_reward": 3.886520874500275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.575,
      "epoch": 1.214555765595463,
      "format_reward": 0.0,
      "grad_norm": 3.3710083961486816,
      "image_reward": 0.2646331787109375,
      "kl": 0.1121824998408556,
      "learning_rate": 5e-06,
      "loss": 0.1077,
      "reward": 0.6361587151885033,
      "reward_std": 0.6423972092568875,
      "rewards/reward_func": 0.6361587151885033,
      "step": 5140,
      "toxic_reward": 4.181642347574234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.625,
      "epoch": 1.2169187145557656,
      "format_reward": 0.0,
      "grad_norm": 1.4757941961288452,
      "image_reward": 0.2700469970703125,
      "kl": 0.808637504093349,
      "learning_rate": 5e-06,
      "loss": 0.0188,
      "reward": 0.2734032437205315,
      "reward_std": 0.8411962412297725,
      "rewards/reward_func": 0.2734032437205315,
      "step": 5150,
      "toxic_reward": 3.6458971202373505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.725,
      "epoch": 1.2192816635160681,
      "format_reward": -0.25,
      "grad_norm": 0.66521817445755,
      "image_reward": 0.2667388916015625,
      "kl": 0.4425561033189297,
      "learning_rate": 5e-06,
      "loss": -0.0435,
      "reward": 0.35035309493541716,
      "reward_std": 1.6248657763004304,
      "rewards/reward_func": 0.35035309493541716,
      "step": 5160,
      "toxic_reward": 3.7190463662147524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.475,
      "epoch": 1.2216446124763705,
      "format_reward": 0.0,
      "grad_norm": 8.705077171325684,
      "image_reward": 0.259429931640625,
      "kl": 0.7811562133952975,
      "learning_rate": 5e-06,
      "loss": 0.0244,
      "reward": 0.3771729826927185,
      "reward_std": 1.326733610033989,
      "rewards/reward_func": 0.3771729826927185,
      "step": 5170,
      "toxic_reward": 3.6760028123855593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.975,
      "epoch": 1.224007561436673,
      "format_reward": 0.0,
      "grad_norm": 14.75157356262207,
      "image_reward": 0.22822214663028717,
      "kl": 45.12481062971055,
      "learning_rate": 5e-06,
      "loss": -0.0908,
      "reward": 0.6647323310375214,
      "reward_std": 1.0134072445333004,
      "rewards/reward_func": 0.6647323310375214,
      "step": 5180,
      "toxic_reward": 4.210642290115357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.4,
      "epoch": 1.2263705103969755,
      "format_reward": -0.25,
      "grad_norm": 13.71264934539795,
      "image_reward": 0.23838348388671876,
      "kl": 0.2953592788428068,
      "learning_rate": 5e-06,
      "loss": -0.0801,
      "reward": 0.4181412994861603,
      "reward_std": 1.2768891528248787,
      "rewards/reward_func": 0.4181412994861603,
      "step": 5190,
      "toxic_reward": 4.502067589759827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.625,
      "epoch": 1.2287334593572778,
      "format_reward": 0.0,
      "grad_norm": 7.2092390060424805,
      "image_reward": 0.2596160888671875,
      "kl": 0.15071408227086067,
      "learning_rate": 5e-06,
      "loss": 0.0359,
      "reward": 0.46884081363677976,
      "reward_std": 0.9004301078617573,
      "rewards/reward_func": 0.46884081363677976,
      "step": 5200,
      "toxic_reward": 4.537938523292541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.55,
      "epoch": 1.2310964083175804,
      "format_reward": 0.0,
      "grad_norm": 1.4807243347167969,
      "image_reward": 0.25061492919921874,
      "kl": 0.39518592432141303,
      "learning_rate": 5e-06,
      "loss": 0.0062,
      "reward": 0.7565897464752197,
      "reward_std": 0.6514241144061088,
      "rewards/reward_func": 0.7565897464752197,
      "step": 5210,
      "toxic_reward": 4.779706335067749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.15,
      "epoch": 1.2334593572778827,
      "format_reward": 0.0,
      "grad_norm": 12.918940544128418,
      "image_reward": 0.266143798828125,
      "kl": 1.6246791556477547,
      "learning_rate": 5e-06,
      "loss": -0.0669,
      "reward": 1.0145411103963853,
      "reward_std": 0.7731746513396501,
      "rewards/reward_func": 1.0145411103963853,
      "step": 5220,
      "toxic_reward": 3.9364122271537783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.8,
      "epoch": 1.2358223062381852,
      "format_reward": -0.5,
      "grad_norm": 8.1648530960083,
      "image_reward": 0.256298828125,
      "kl": 0.3491713672876358,
      "learning_rate": 5e-06,
      "loss": -0.0294,
      "reward": 0.18980904817581176,
      "reward_std": 1.4395622819662095,
      "rewards/reward_func": 0.18980904817581176,
      "step": 5230,
      "toxic_reward": 4.15175496339798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.675,
      "epoch": 1.2381852551984878,
      "format_reward": -0.25,
      "grad_norm": 7.493502140045166,
      "image_reward": 0.2956329345703125,
      "kl": 3.9262495055794715,
      "learning_rate": 5e-06,
      "loss": 0.0129,
      "reward": 0.19967559576034546,
      "reward_std": 1.4724704299122096,
      "rewards/reward_func": 0.19967559576034546,
      "step": 5240,
      "toxic_reward": 3.676086974143982
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.95,
      "epoch": 1.24054820415879,
      "format_reward": 0.0,
      "grad_norm": 7.836026668548584,
      "image_reward": 0.262060546875,
      "kl": 0.5677594847977161,
      "learning_rate": 5e-06,
      "loss": -0.0137,
      "reward": 1.0836671590805054,
      "reward_std": 0.9185017041862011,
      "rewards/reward_func": 1.0836671590805054,
      "step": 5250,
      "toxic_reward": 4.442173409461975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.925,
      "epoch": 1.2429111531190926,
      "format_reward": -0.25,
      "grad_norm": 17.290130615234375,
      "image_reward": 0.256195068359375,
      "kl": 0.3261503577232361,
      "learning_rate": 5e-06,
      "loss": -0.0665,
      "reward": 0.4270883619785309,
      "reward_std": 1.5899662226438522,
      "rewards/reward_func": 0.4270883619785309,
      "step": 5260,
      "toxic_reward": 3.6384164452552796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.425,
      "epoch": 1.2452741020793952,
      "format_reward": 0.0,
      "grad_norm": 19.655460357666016,
      "image_reward": 0.271282958984375,
      "kl": 1.0409250572323798,
      "learning_rate": 5e-06,
      "loss": -0.0411,
      "reward": 0.7604422211647034,
      "reward_std": 0.6456888254731894,
      "rewards/reward_func": 0.7604422211647034,
      "step": 5270,
      "toxic_reward": 3.7677977979183197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 59.775,
      "epoch": 1.2476370510396975,
      "format_reward": -0.25,
      "grad_norm": 0.5878366827964783,
      "image_reward": 0.2589070647954941,
      "kl": 0.16051149740815163,
      "learning_rate": 5e-06,
      "loss": 0.0852,
      "reward": 0.39556344896554946,
      "reward_std": 1.1551922081038355,
      "rewards/reward_func": 0.39556344896554946,
      "step": 5280,
      "toxic_reward": 3.390644001960754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.275,
      "epoch": 1.25,
      "format_reward": 0.0,
      "grad_norm": 8.206055641174316,
      "image_reward": 0.275860595703125,
      "kl": 0.4969006285071373,
      "learning_rate": 5e-06,
      "loss": -0.115,
      "reward": 0.4857667863368988,
      "reward_std": 0.8739027962088585,
      "rewards/reward_func": 0.4857667863368988,
      "step": 5290,
      "toxic_reward": 4.016790902614593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.7,
      "epoch": 1.2523629489603025,
      "format_reward": 0.0,
      "grad_norm": 3.513704299926758,
      "image_reward": 0.24937744140625,
      "kl": 0.14417755380272865,
      "learning_rate": 5e-06,
      "loss": 0.0317,
      "reward": 0.48732776641845704,
      "reward_std": 0.8942459903657436,
      "rewards/reward_func": 0.48732776641845704,
      "step": 5300,
      "toxic_reward": 4.074605274200439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.7,
      "epoch": 1.2547258979206048,
      "format_reward": 0.0,
      "grad_norm": 3.694108724594116,
      "image_reward": 0.2640960693359375,
      "kl": 0.21989786028862,
      "learning_rate": 5e-06,
      "loss": 0.0552,
      "reward": 0.20011116266250611,
      "reward_std": 0.9783342686016112,
      "rewards/reward_func": 0.20011116266250611,
      "step": 5310,
      "toxic_reward": 3.337161436676979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 61.15,
      "epoch": 1.2570888468809074,
      "format_reward": -0.25,
      "grad_norm": 1.8417941331863403,
      "image_reward": 0.23155619353055953,
      "kl": 4.248336365818977,
      "learning_rate": 5e-06,
      "loss": 0.0877,
      "reward": 0.23556498885154725,
      "reward_std": 0.9007356996648014,
      "rewards/reward_func": 0.23556498885154725,
      "step": 5320,
      "toxic_reward": 4.5890906810760494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.475,
      "epoch": 1.2594517958412097,
      "format_reward": 0.0,
      "grad_norm": 4.823044300079346,
      "image_reward": 0.2513310745358467,
      "kl": 4.037289990484714,
      "learning_rate": 5e-06,
      "loss": 0.082,
      "reward": 0.8450765609741211,
      "reward_std": 0.8255521267652511,
      "rewards/reward_func": 0.8450765609741211,
      "step": 5330,
      "toxic_reward": 4.287392568588257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.975,
      "epoch": 1.2618147448015122,
      "format_reward": 0.0,
      "grad_norm": 4.315946578979492,
      "image_reward": 0.2459075927734375,
      "kl": 0.3113373316824436,
      "learning_rate": 5e-06,
      "loss": 0.0475,
      "reward": 0.645756970345974,
      "reward_std": 0.7255122657865286,
      "rewards/reward_func": 0.645756970345974,
      "step": 5340,
      "toxic_reward": 4.189401495456695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.7,
      "epoch": 1.2641776937618148,
      "format_reward": 0.0,
      "grad_norm": 3.4027810096740723,
      "image_reward": 0.2551483154296875,
      "kl": 0.4323126286268234,
      "learning_rate": 5e-06,
      "loss": -0.0048,
      "reward": 0.5660954803228379,
      "reward_std": 0.6791210256516933,
      "rewards/reward_func": 0.5660954803228379,
      "step": 5350,
      "toxic_reward": 3.2965795576572416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.55,
      "epoch": 1.2665406427221173,
      "format_reward": 0.0,
      "grad_norm": 1.9737337827682495,
      "image_reward": 0.2604766845703125,
      "kl": 0.8922965943813324,
      "learning_rate": 5e-06,
      "loss": 0.0886,
      "reward": 0.37445068359375,
      "reward_std": 0.7902419693768025,
      "rewards/reward_func": 0.37445068359375,
      "step": 5360,
      "toxic_reward": 3.6073597192764284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.975,
      "epoch": 1.2689035916824196,
      "format_reward": -0.25,
      "grad_norm": 5.368748188018799,
      "image_reward": 0.2573964446783066,
      "kl": 0.8937133550643921,
      "learning_rate": 5e-06,
      "loss": 0.0345,
      "reward": 1.1729332506656647,
      "reward_std": 1.3139135614037514,
      "rewards/reward_func": 1.1729332506656647,
      "step": 5370,
      "toxic_reward": 4.428536581993103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.375,
      "epoch": 1.2712665406427222,
      "format_reward": 0.0,
      "grad_norm": 2.3669607639312744,
      "image_reward": 0.242498779296875,
      "kl": 0.6131832510232925,
      "learning_rate": 5e-06,
      "loss": -0.1065,
      "reward": 0.3124019861221313,
      "reward_std": 0.8398781210184098,
      "rewards/reward_func": 0.3124019861221313,
      "step": 5380,
      "toxic_reward": 3.9513532400131224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 62.325,
      "epoch": 1.2736294896030245,
      "format_reward": 0.0,
      "grad_norm": 3.6428773403167725,
      "image_reward": 0.2564788818359375,
      "kl": 0.983223095536232,
      "learning_rate": 5e-06,
      "loss": -0.0332,
      "reward": 0.8021515548229218,
      "reward_std": 0.8680705142207443,
      "rewards/reward_func": 0.8021515548229218,
      "step": 5390,
      "toxic_reward": 3.7623249292373657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.375,
      "epoch": 1.275992438563327,
      "format_reward": 0.0,
      "grad_norm": 22.341930389404297,
      "image_reward": 0.25401458740234373,
      "kl": 0.7686945527791977,
      "learning_rate": 5e-06,
      "loss": 0.021,
      "reward": 0.18261390328407287,
      "reward_std": 0.39404432671144607,
      "rewards/reward_func": 0.18261390328407287,
      "step": 5400,
      "toxic_reward": 3.986022639274597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.075,
      "epoch": 1.2783553875236295,
      "format_reward": -0.5,
      "grad_norm": 11.878053665161133,
      "image_reward": 0.24981587678194045,
      "kl": 0.4886711150407791,
      "learning_rate": 5e-06,
      "loss": -0.0214,
      "reward": 0.06913218498229981,
      "reward_std": 1.5105109971016646,
      "rewards/reward_func": 0.06913218498229981,
      "step": 5410,
      "toxic_reward": 4.118505048751831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.65,
      "epoch": 1.280718336483932,
      "format_reward": -0.25,
      "grad_norm": 7.851999759674072,
      "image_reward": 0.25881449431180953,
      "kl": 0.6457854598760605,
      "learning_rate": 5e-06,
      "loss": 0.0958,
      "reward": 0.12992151379585265,
      "reward_std": 1.303325356543064,
      "rewards/reward_func": 0.12992151379585265,
      "step": 5420,
      "toxic_reward": 4.377404046058655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.9,
      "epoch": 1.2830812854442344,
      "format_reward": -0.25,
      "grad_norm": 25.7547550201416,
      "image_reward": 0.2694793701171875,
      "kl": 1.677524197101593,
      "learning_rate": 5e-06,
      "loss": 0.0859,
      "reward": 0.34250465631484983,
      "reward_std": 1.0538076907396317,
      "rewards/reward_func": 0.34250465631484983,
      "step": 5430,
      "toxic_reward": 4.271343016624451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.875,
      "epoch": 1.285444234404537,
      "format_reward": 0.0,
      "grad_norm": 1.9158964157104492,
      "image_reward": 0.2666290283203125,
      "kl": 0.5966441169381141,
      "learning_rate": 5e-06,
      "loss": -0.0566,
      "reward": 0.4593892157077789,
      "reward_std": 0.6576637156307698,
      "rewards/reward_func": 0.4593892157077789,
      "step": 5440,
      "toxic_reward": 4.204905700683594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.525,
      "epoch": 1.2878071833648392,
      "format_reward": 0.0,
      "grad_norm": 1.6007134914398193,
      "image_reward": 0.2621429443359375,
      "kl": 1.1984394997358323,
      "learning_rate": 5e-06,
      "loss": -0.0486,
      "reward": 0.25084500312805175,
      "reward_std": 1.5825427711009978,
      "rewards/reward_func": 0.25084500312805175,
      "step": 5450,
      "toxic_reward": 3.685545027256012
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.45,
      "epoch": 1.2901701323251418,
      "format_reward": -0.25,
      "grad_norm": 25.30818748474121,
      "image_reward": 0.2647552490234375,
      "kl": 6.056701734662056,
      "learning_rate": 5e-06,
      "loss": 0.0599,
      "reward": 0.25077282190322875,
      "reward_std": 1.234234382212162,
      "rewards/reward_func": 0.25077282190322875,
      "step": 5460,
      "toxic_reward": 4.593598937988281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.825,
      "epoch": 1.2925330812854443,
      "format_reward": 0.0,
      "grad_norm": 16.34868812561035,
      "image_reward": 0.2645416259765625,
      "kl": 1.6588621526956557,
      "learning_rate": 5e-06,
      "loss": 0.0054,
      "reward": 0.5463581264019013,
      "reward_std": 0.7020838841795921,
      "rewards/reward_func": 0.5463581264019013,
      "step": 5470,
      "toxic_reward": 4.367759561538696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 57.5,
      "epoch": 1.2948960302457466,
      "format_reward": 0.0,
      "grad_norm": 19.92365264892578,
      "image_reward": 0.235223388671875,
      "kl": 4.412905436754227,
      "learning_rate": 5e-06,
      "loss": -0.0277,
      "reward": 0.301082968711853,
      "reward_std": 0.5573954021558165,
      "rewards/reward_func": 0.301082968711853,
      "step": 5480,
      "toxic_reward": 4.563822269439697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.45,
      "epoch": 1.2972589792060492,
      "format_reward": -0.25,
      "grad_norm": 15.039924621582031,
      "image_reward": 0.2686960846185684,
      "kl": 1.0792655169963836,
      "learning_rate": 5e-06,
      "loss": -0.0051,
      "reward": -0.10340776294469833,
      "reward_std": 1.0451893661171199,
      "rewards/reward_func": -0.10340776294469833,
      "step": 5490,
      "toxic_reward": 3.9914595246315003
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.675,
      "epoch": 1.2996219281663515,
      "format_reward": 0.0,
      "grad_norm": 20.686878204345703,
      "image_reward": 0.236834716796875,
      "kl": 0.6705092936754227,
      "learning_rate": 5e-06,
      "loss": -0.0657,
      "reward": 0.7118561029434204,
      "reward_std": 0.6682203419506549,
      "rewards/reward_func": 0.7118561029434204,
      "step": 5500,
      "toxic_reward": 4.56234884262085
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.45,
      "epoch": 1.301984877126654,
      "format_reward": -0.25,
      "grad_norm": 9.817633628845215,
      "image_reward": 0.2553232818841934,
      "kl": 0.9995080977678299,
      "learning_rate": 5e-06,
      "loss": -0.1257,
      "reward": 0.27432467341423034,
      "reward_std": 1.3118387231603266,
      "rewards/reward_func": 0.27432467341423034,
      "step": 5510,
      "toxic_reward": 4.407331418991089
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.6,
      "epoch": 1.3043478260869565,
      "format_reward": 0.0,
      "grad_norm": 11.929862976074219,
      "image_reward": 0.2562835693359375,
      "kl": 11.621018621325494,
      "learning_rate": 5e-06,
      "loss": 0.0747,
      "reward": 0.29978330433368683,
      "reward_std": 0.5768878096714616,
      "rewards/reward_func": 0.29978330433368683,
      "step": 5520,
      "toxic_reward": 3.9843369722366333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.375,
      "epoch": 1.306710775047259,
      "format_reward": -0.5,
      "grad_norm": 54.56308364868164,
      "image_reward": 0.25370279848575594,
      "kl": 15.30106150507927,
      "learning_rate": 5e-06,
      "loss": 0.059,
      "reward": -0.3965187072753906,
      "reward_std": 1.5167736381292343,
      "rewards/reward_func": -0.3965187072753906,
      "step": 5530,
      "toxic_reward": 4.435292959213257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.35,
      "epoch": 1.3090737240075614,
      "format_reward": 0.0,
      "grad_norm": 3.027195930480957,
      "image_reward": 0.30064697265625,
      "kl": 1.0911591410636903,
      "learning_rate": 5e-06,
      "loss": -0.0513,
      "reward": 0.3132080137729645,
      "reward_std": 0.5429811116307974,
      "rewards/reward_func": 0.3132080137729645,
      "step": 5540,
      "toxic_reward": 4.454129576683044
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.2,
      "epoch": 1.311436672967864,
      "format_reward": 0.0,
      "grad_norm": 14.916865348815918,
      "image_reward": 0.2515268951654434,
      "kl": 5.700361841917038,
      "learning_rate": 5e-06,
      "loss": 0.0014,
      "reward": 0.014350098371505738,
      "reward_std": 0.48063138537108896,
      "rewards/reward_func": 0.014350098371505738,
      "step": 5550,
      "toxic_reward": 4.399230480194092
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.975,
      "epoch": 1.3137996219281662,
      "format_reward": -0.25,
      "grad_norm": 17.978458404541016,
      "image_reward": 0.24458109587430954,
      "kl": 4.160827812552452,
      "learning_rate": 5e-06,
      "loss": -0.0048,
      "reward": 0.29164408445358275,
      "reward_std": 0.9224476981908083,
      "rewards/reward_func": 0.29164408445358275,
      "step": 5560,
      "toxic_reward": 4.361080431938172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.7,
      "epoch": 1.3161625708884688,
      "format_reward": 0.0,
      "grad_norm": 2.0734810829162598,
      "image_reward": 0.248583984375,
      "kl": 2.431508493423462,
      "learning_rate": 5e-06,
      "loss": -0.0346,
      "reward": 0.49899758100509645,
      "reward_std": 0.9045591181144118,
      "rewards/reward_func": 0.49899758100509645,
      "step": 5570,
      "toxic_reward": 4.267354512214661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.675,
      "epoch": 1.3185255198487713,
      "format_reward": -0.25,
      "grad_norm": 7.6361165046691895,
      "image_reward": 0.27701314240694047,
      "kl": 1.473704105615616,
      "learning_rate": 5e-06,
      "loss": 0.0156,
      "reward": 0.5548757612705231,
      "reward_std": 0.9885425483807921,
      "rewards/reward_func": 0.5548757612705231,
      "step": 5580,
      "toxic_reward": 4.615298962593078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.35,
      "epoch": 1.3208884688090738,
      "format_reward": 0.0,
      "grad_norm": 8.7377290725708,
      "image_reward": 0.2538330078125,
      "kl": 197.4472616136074,
      "learning_rate": 5e-06,
      "loss": 0.1633,
      "reward": 0.6491668224334717,
      "reward_std": 0.6371353514492512,
      "rewards/reward_func": 0.6491668224334717,
      "step": 5590,
      "toxic_reward": 4.438857316970825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.575,
      "epoch": 1.3232514177693762,
      "format_reward": 0.0,
      "grad_norm": 16.95717430114746,
      "image_reward": 0.23577117919921875,
      "kl": 3.680394399166107,
      "learning_rate": 5e-06,
      "loss": -0.0084,
      "reward": 0.1440478801727295,
      "reward_std": 0.4425256311893463,
      "rewards/reward_func": 0.1440478801727295,
      "step": 5600,
      "toxic_reward": 4.603517079353333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.25,
      "epoch": 1.3256143667296787,
      "format_reward": 0.0,
      "grad_norm": 12.732531547546387,
      "image_reward": 0.2568023681640625,
      "kl": 2.256978714466095,
      "learning_rate": 5e-06,
      "loss": 0.0219,
      "reward": 0.040336894989013675,
      "reward_std": 0.6101976454257965,
      "rewards/reward_func": 0.040336894989013675,
      "step": 5610,
      "toxic_reward": 4.473554587364196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.75,
      "epoch": 1.327977315689981,
      "format_reward": 0.0,
      "grad_norm": 14.701716423034668,
      "image_reward": 0.252197265625,
      "kl": 8.072559344768525,
      "learning_rate": 5e-06,
      "loss": 0.0803,
      "reward": 0.8639614999294281,
      "reward_std": 1.0928052112460136,
      "rewards/reward_func": 0.8639614999294281,
      "step": 5620,
      "toxic_reward": 3.9843992233276366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.8,
      "epoch": 1.3303402646502835,
      "format_reward": 0.0,
      "grad_norm": 10.12986946105957,
      "image_reward": 0.27263641357421875,
      "kl": 3.3592694640159606,
      "learning_rate": 5e-06,
      "loss": 0.0652,
      "reward": 0.7886571228504181,
      "reward_std": 1.0737986475229264,
      "rewards/reward_func": 0.7886571228504181,
      "step": 5630,
      "toxic_reward": 3.668324041366577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.95,
      "epoch": 1.332703213610586,
      "format_reward": 0.0,
      "grad_norm": 23.82711410522461,
      "image_reward": 0.2702301025390625,
      "kl": 12.466990399360657,
      "learning_rate": 5e-06,
      "loss": -0.0401,
      "reward": 0.7557259559631347,
      "reward_std": 0.9376067817211151,
      "rewards/reward_func": 0.7557259559631347,
      "step": 5640,
      "toxic_reward": 4.100273895263672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.625,
      "epoch": 1.3350661625708884,
      "format_reward": 0.0,
      "grad_norm": 7.783689975738525,
      "image_reward": 0.2736572265625,
      "kl": 9.604325413703918,
      "learning_rate": 5e-06,
      "loss": 0.0135,
      "reward": 0.21887901425361633,
      "reward_std": 0.41371094444766643,
      "rewards/reward_func": 0.21887901425361633,
      "step": 5650,
      "toxic_reward": 4.314438569545746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.125,
      "epoch": 1.337429111531191,
      "format_reward": 0.0,
      "grad_norm": 8.773420333862305,
      "image_reward": 0.2513946533203125,
      "kl": 9.296885073184967,
      "learning_rate": 5e-06,
      "loss": -0.095,
      "reward": 1.1378837168216704,
      "reward_std": 0.818750386312604,
      "rewards/reward_func": 1.1378837168216704,
      "step": 5660,
      "toxic_reward": 4.522894716262817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.9,
      "epoch": 1.3397920604914935,
      "format_reward": -0.25,
      "grad_norm": 14.245692253112793,
      "image_reward": 0.24806925505399705,
      "kl": 6.753875517845154,
      "learning_rate": 5e-06,
      "loss": 0.021,
      "reward": 0.6894584268331527,
      "reward_std": 1.542804090678692,
      "rewards/reward_func": 0.6894584268331527,
      "step": 5670,
      "toxic_reward": 3.9723486423492433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.075,
      "epoch": 1.3421550094517958,
      "format_reward": -0.25,
      "grad_norm": 24.408653259277344,
      "image_reward": 0.25640462189912794,
      "kl": 11.201071047782898,
      "learning_rate": 5e-06,
      "loss": -0.0492,
      "reward": -0.00045427381992340087,
      "reward_std": 1.027926566079259,
      "rewards/reward_func": -0.00045427381992340087,
      "step": 5680,
      "toxic_reward": 3.891611325740814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.65,
      "epoch": 1.3445179584120983,
      "format_reward": 0.0,
      "grad_norm": 1.8934930562973022,
      "image_reward": 0.25365397036075593,
      "kl": 4.947464096546173,
      "learning_rate": 5e-06,
      "loss": -0.0784,
      "reward": 0.5487861603498458,
      "reward_std": 0.7702463563531637,
      "rewards/reward_func": 0.5487861603498458,
      "step": 5690,
      "toxic_reward": 3.977373069524765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.5,
      "epoch": 1.3468809073724008,
      "format_reward": -0.25,
      "grad_norm": 9.322084426879883,
      "image_reward": 0.27449544221162797,
      "kl": 2.6174604117870333,
      "learning_rate": 5e-06,
      "loss": -0.0094,
      "reward": 0.39208410382270814,
      "reward_std": 1.61805320084095,
      "rewards/reward_func": 0.39208410382270814,
      "step": 5700,
      "toxic_reward": 4.174729800224304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.25,
      "epoch": 1.3492438563327032,
      "format_reward": 0.0,
      "grad_norm": 12.4689302444458,
      "image_reward": 0.2571044921875,
      "kl": 3.102731728553772,
      "learning_rate": 5e-06,
      "loss": -0.0306,
      "reward": 0.24812114238739014,
      "reward_std": 0.6699782099574805,
      "rewards/reward_func": 0.24812114238739014,
      "step": 5710,
      "toxic_reward": 3.692942750453949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.425,
      "epoch": 1.3516068052930057,
      "format_reward": 0.0,
      "grad_norm": 16.464384078979492,
      "image_reward": 0.2592987060546875,
      "kl": 41.42341262102127,
      "learning_rate": 5e-06,
      "loss": -0.1787,
      "reward": 0.9784101039171219,
      "reward_std": 1.2197245783172548,
      "rewards/reward_func": 0.9784101039171219,
      "step": 5720,
      "toxic_reward": 3.5939176797866823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.775,
      "epoch": 1.353969754253308,
      "format_reward": 0.0,
      "grad_norm": 14.272177696228027,
      "image_reward": 0.24337158203125,
      "kl": 3.5139986366033553,
      "learning_rate": 5e-06,
      "loss": -0.0502,
      "reward": 0.3250808596611023,
      "reward_std": 0.6109479434788228,
      "rewards/reward_func": 0.3250808596611023,
      "step": 5730,
      "toxic_reward": 4.485757279396057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.6,
      "epoch": 1.3563327032136105,
      "format_reward": -0.25,
      "grad_norm": 8.131665229797363,
      "image_reward": 0.2514506012201309,
      "kl": 5.592804127931595,
      "learning_rate": 5e-06,
      "loss": -0.015,
      "reward": 0.3052162408828735,
      "reward_std": 1.201428510248661,
      "rewards/reward_func": 0.3052162408828735,
      "step": 5740,
      "toxic_reward": 4.2779217004776005
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.275,
      "epoch": 1.358695652173913,
      "format_reward": -0.5,
      "grad_norm": 15.648195266723633,
      "image_reward": 0.266064453125,
      "kl": 1.6513773769140243,
      "learning_rate": 5e-06,
      "loss": 0.0092,
      "reward": -0.20032901763916017,
      "reward_std": 1.7222102746367454,
      "rewards/reward_func": -0.20032901763916017,
      "step": 5750,
      "toxic_reward": 4.259865856170654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.0,
      "epoch": 1.3610586011342156,
      "format_reward": 0.0,
      "grad_norm": 10.893685340881348,
      "image_reward": 0.2588653564453125,
      "kl": 4.073341834545135,
      "learning_rate": 5e-06,
      "loss": 0.013,
      "reward": 0.916795802116394,
      "reward_std": 0.8524092853069305,
      "rewards/reward_func": 0.916795802116394,
      "step": 5760,
      "toxic_reward": 4.560049152374267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.75,
      "epoch": 1.363421550094518,
      "format_reward": 0.0,
      "grad_norm": 3.932856798171997,
      "image_reward": 0.2459381103515625,
      "kl": 2.5305844336748122,
      "learning_rate": 5e-06,
      "loss": 0.0338,
      "reward": 0.5017880856990814,
      "reward_std": 0.7364757396280766,
      "rewards/reward_func": 0.5017880856990814,
      "step": 5770,
      "toxic_reward": 4.69781801700592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.45,
      "epoch": 1.3657844990548205,
      "format_reward": 0.0,
      "grad_norm": 1.3704707622528076,
      "image_reward": 0.2677764892578125,
      "kl": 1.8369466960430145,
      "learning_rate": 5e-06,
      "loss": 0.107,
      "reward": 0.7046410620212555,
      "reward_std": 0.9321951523423195,
      "rewards/reward_func": 0.7046410620212555,
      "step": 5780,
      "toxic_reward": 4.073530220985413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.8,
      "epoch": 1.3681474480151228,
      "format_reward": -0.25,
      "grad_norm": 2.7950003147125244,
      "image_reward": 0.268048095703125,
      "kl": 3.3737578272819517,
      "learning_rate": 5e-06,
      "loss": 0.0251,
      "reward": 0.1121946096420288,
      "reward_std": 1.2333336278796196,
      "rewards/reward_func": 0.1121946096420288,
      "step": 5790,
      "toxic_reward": 4.301294279098511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.9,
      "epoch": 1.3705103969754253,
      "format_reward": -0.25,
      "grad_norm": 10.600517272949219,
      "image_reward": 0.24301045686006545,
      "kl": 5.24166065454483,
      "learning_rate": 5e-06,
      "loss": -0.0188,
      "reward": -0.04525191783905029,
      "reward_std": 1.1580330106429755,
      "rewards/reward_func": -0.04525191783905029,
      "step": 5800,
      "toxic_reward": 3.95846186876297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.6,
      "epoch": 1.3728733459357279,
      "format_reward": -0.5,
      "grad_norm": 22.423450469970703,
      "image_reward": 0.25951487123966216,
      "kl": 12.250067234039307,
      "learning_rate": 5e-06,
      "loss": 0.0091,
      "reward": 0.0036635279655456545,
      "reward_std": 1.6421116095036268,
      "rewards/reward_func": 0.0036635279655456545,
      "step": 5810,
      "toxic_reward": 4.407265400886535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.825,
      "epoch": 1.3752362948960302,
      "format_reward": 0.0,
      "grad_norm": 3.5839083194732666,
      "image_reward": 0.2497711181640625,
      "kl": 7.638963532447815,
      "learning_rate": 5e-06,
      "loss": -0.0623,
      "reward": 0.36217689514160156,
      "reward_std": 1.057050895690918,
      "rewards/reward_func": 0.36217689514160156,
      "step": 5820,
      "toxic_reward": 3.8359474897384644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.075,
      "epoch": 1.3775992438563327,
      "format_reward": 0.0,
      "grad_norm": 18.257360458374023,
      "image_reward": 0.242034912109375,
      "kl": 1406.6461040258407,
      "learning_rate": 5e-06,
      "loss": 0.3409,
      "reward": 0.35478733479976654,
      "reward_std": 0.5706452172249555,
      "rewards/reward_func": 0.35478733479976654,
      "step": 5830,
      "toxic_reward": 3.5973093271255494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.55,
      "epoch": 1.3799621928166352,
      "format_reward": 0.0,
      "grad_norm": 4.36010217666626,
      "image_reward": 0.24814300537109374,
      "kl": 117.68144319057464,
      "learning_rate": 5e-06,
      "loss": 0.0637,
      "reward": 0.3609376668930054,
      "reward_std": 0.6294937739614397,
      "rewards/reward_func": 0.3609376668930054,
      "step": 5840,
      "toxic_reward": 4.1664423704147335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.65,
      "epoch": 1.3823251417769375,
      "format_reward": 0.0,
      "grad_norm": 14.234587669372559,
      "image_reward": 0.254736328125,
      "kl": 4.648911118507385,
      "learning_rate": 5e-06,
      "loss": -0.0132,
      "reward": -0.4629164904356003,
      "reward_std": 0.8635658169165253,
      "rewards/reward_func": -0.4629164904356003,
      "step": 5850,
      "toxic_reward": 3.804247868061066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.825,
      "epoch": 1.38468809073724,
      "format_reward": -0.25,
      "grad_norm": 9.249091148376465,
      "image_reward": 0.25889790803194046,
      "kl": 8.51909922361374,
      "learning_rate": 5e-06,
      "loss": 0.0656,
      "reward": 0.20021165013313294,
      "reward_std": 1.1463583020493389,
      "rewards/reward_func": 0.20021165013313294,
      "step": 5860,
      "toxic_reward": 4.298932027816773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.2,
      "epoch": 1.3870510396975426,
      "format_reward": 0.0,
      "grad_norm": 2.6423728466033936,
      "image_reward": 0.2569976806640625,
      "kl": 114.30452468395234,
      "learning_rate": 5e-06,
      "loss": 0.0235,
      "reward": 0.7682538509368897,
      "reward_std": 0.9905061937868596,
      "rewards/reward_func": 0.7682538509368897,
      "step": 5870,
      "toxic_reward": 4.355731654167175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 57.75,
      "epoch": 1.389413988657845,
      "format_reward": 0.0,
      "grad_norm": 6.193624496459961,
      "image_reward": 0.244573974609375,
      "kl": 27.17574143409729,
      "learning_rate": 5e-06,
      "loss": 0.0466,
      "reward": 0.4181258499622345,
      "reward_std": 0.7019964678213,
      "rewards/reward_func": 0.4181258499622345,
      "step": 5880,
      "toxic_reward": 4.30544638633728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.875,
      "epoch": 1.3917769376181475,
      "format_reward": 0.0,
      "grad_norm": 14.119263648986816,
      "image_reward": 0.24458719789981842,
      "kl": 22.515019488334655,
      "learning_rate": 5e-06,
      "loss": 0.022,
      "reward": 0.2917088523507118,
      "reward_std": 0.7304708318784833,
      "rewards/reward_func": 0.2917088523507118,
      "step": 5890,
      "toxic_reward": 4.011698079109192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.95,
      "epoch": 1.3941398865784498,
      "format_reward": 0.0,
      "grad_norm": 203.53358459472656,
      "image_reward": 0.266705322265625,
      "kl": 130.7643344759941,
      "learning_rate": 5e-06,
      "loss": -0.0925,
      "reward": 0.8561399459838868,
      "reward_std": 0.7673989269882441,
      "rewards/reward_func": 0.8561399459838868,
      "step": 5900,
      "toxic_reward": 4.3426886081695555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.25,
      "epoch": 1.3965028355387523,
      "format_reward": 0.0,
      "grad_norm": 1.8555150032043457,
      "image_reward": 0.26331787109375,
      "kl": 25.51781210899353,
      "learning_rate": 5e-06,
      "loss": -0.0759,
      "reward": 0.14676390141248702,
      "reward_std": 0.31099242605268956,
      "rewards/reward_func": 0.14676390141248702,
      "step": 5910,
      "toxic_reward": 4.393804085254669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.375,
      "epoch": 1.3988657844990549,
      "format_reward": 0.0,
      "grad_norm": 1.794382095336914,
      "image_reward": 0.2638519287109375,
      "kl": 75.26498790383339,
      "learning_rate": 5e-06,
      "loss": -0.0105,
      "reward": 0.0749910295009613,
      "reward_std": 0.9545040905475617,
      "rewards/reward_func": 0.0749910295009613,
      "step": 5920,
      "toxic_reward": 4.043283843994141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.6,
      "epoch": 1.4012287334593574,
      "format_reward": 0.0,
      "grad_norm": 10.70461654663086,
      "image_reward": 0.26204325407743456,
      "kl": 6.424372181296349,
      "learning_rate": 5e-06,
      "loss": -0.0192,
      "reward": 0.3424019992351532,
      "reward_std": 0.8532586313784123,
      "rewards/reward_func": 0.3424019992351532,
      "step": 5930,
      "toxic_reward": 3.697358027100563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.7,
      "epoch": 1.4035916824196597,
      "format_reward": 0.0,
      "grad_norm": 15.443364143371582,
      "image_reward": 0.25388997346162795,
      "kl": 2.7157889783382414,
      "learning_rate": 5e-06,
      "loss": 0.0194,
      "reward": 0.8944644808769227,
      "reward_std": 0.907353313267231,
      "rewards/reward_func": 0.8944644808769227,
      "step": 5940,
      "toxic_reward": 4.2960577487945555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.625,
      "epoch": 1.4059546313799622,
      "format_reward": 0.0,
      "grad_norm": 9.80057144165039,
      "image_reward": 0.2643798828125,
      "kl": 3.2323968172073365,
      "learning_rate": 5e-06,
      "loss": -0.0714,
      "reward": 0.7592375218868256,
      "reward_std": 1.0486908692866563,
      "rewards/reward_func": 0.7592375218868256,
      "step": 5950,
      "toxic_reward": 4.236204934120178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.45,
      "epoch": 1.4083175803402646,
      "format_reward": 0.0,
      "grad_norm": 32.608253479003906,
      "image_reward": 0.2897979736328125,
      "kl": 0.868326199054718,
      "learning_rate": 5e-06,
      "loss": 0.0072,
      "reward": 0.13819260597229005,
      "reward_std": 0.9927060969173909,
      "rewards/reward_func": 0.13819260597229005,
      "step": 5960,
      "toxic_reward": 4.137164163589477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.875,
      "epoch": 1.410680529300567,
      "format_reward": -0.5,
      "grad_norm": 3.4970862865448,
      "image_reward": 0.254620361328125,
      "kl": 1.8707860291004181,
      "learning_rate": 5e-06,
      "loss": -0.0251,
      "reward": -0.12371634542942048,
      "reward_std": 1.6602010667324065,
      "rewards/reward_func": -0.12371634542942048,
      "step": 5970,
      "toxic_reward": 4.560637950897217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.675,
      "epoch": 1.4130434782608696,
      "format_reward": 0.0,
      "grad_norm": 18.951919555664062,
      "image_reward": 0.258807373046875,
      "kl": 1.7996377795934677,
      "learning_rate": 5e-06,
      "loss": 0.012,
      "reward": 0.7075730919837951,
      "reward_std": 0.9400279764086008,
      "rewards/reward_func": 0.7075730919837951,
      "step": 5980,
      "toxic_reward": 3.758779287338257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.025,
      "epoch": 1.4154064272211722,
      "format_reward": 0.0,
      "grad_norm": 5.1872663497924805,
      "image_reward": 0.2968048095703125,
      "kl": 4.290337887406349,
      "learning_rate": 5e-06,
      "loss": -0.0076,
      "reward": 0.3689495801925659,
      "reward_std": 0.5776140118017793,
      "rewards/reward_func": 0.3689495801925659,
      "step": 5990,
      "toxic_reward": 4.157499670982361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.225,
      "epoch": 1.4177693761814745,
      "format_reward": 0.0,
      "grad_norm": 4.382224082946777,
      "image_reward": 0.24061279296875,
      "kl": 3.217728292942047,
      "learning_rate": 5e-06,
      "loss": -0.0161,
      "reward": 0.37828874588012695,
      "reward_std": 0.3327252045273781,
      "rewards/reward_func": 0.37828874588012695,
      "step": 6000,
      "toxic_reward": 4.625493478775025
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.625,
      "epoch": 1.420132325141777,
      "format_reward": 0.0,
      "grad_norm": 17.742074966430664,
      "image_reward": 0.260565185546875,
      "kl": 3.087987443804741,
      "learning_rate": 5e-06,
      "loss": -0.0098,
      "reward": 0.22000501453876495,
      "reward_std": 0.6759357416536659,
      "rewards/reward_func": 0.22000501453876495,
      "step": 6010,
      "toxic_reward": 4.1144504189491276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.725,
      "epoch": 1.4224952741020793,
      "format_reward": -0.25,
      "grad_norm": 23.140647888183594,
      "image_reward": 0.2770843505859375,
      "kl": 1.3970532178878785,
      "learning_rate": 5e-06,
      "loss": -0.0249,
      "reward": -0.045973950624465944,
      "reward_std": 1.367066621594131,
      "rewards/reward_func": -0.045973950624465944,
      "step": 6020,
      "toxic_reward": 4.268449664115906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.125,
      "epoch": 1.4248582230623819,
      "format_reward": -0.5,
      "grad_norm": 63.63026428222656,
      "image_reward": 0.2500905364751816,
      "kl": 1.5568452209234238,
      "learning_rate": 5e-06,
      "loss": 0.032,
      "reward": 0.36329651772975924,
      "reward_std": 2.2665354389697314,
      "rewards/reward_func": 0.36329651772975924,
      "step": 6030,
      "toxic_reward": 3.9466104745864867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.15,
      "epoch": 1.4272211720226844,
      "format_reward": -0.25,
      "grad_norm": 3.4662349224090576,
      "image_reward": 0.2569427490234375,
      "kl": 2.127922511100769,
      "learning_rate": 5e-06,
      "loss": 0.0478,
      "reward": 0.4287997782230377,
      "reward_std": 1.335706689953804,
      "rewards/reward_func": 0.4287997782230377,
      "step": 6040,
      "toxic_reward": 4.417151093482971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.775,
      "epoch": 1.4295841209829867,
      "format_reward": 0.0,
      "grad_norm": 6.905588626861572,
      "image_reward": 0.2496734619140625,
      "kl": 2.1467004269361496,
      "learning_rate": 5e-06,
      "loss": -0.1361,
      "reward": 0.6416638314723968,
      "reward_std": 0.6212250446900726,
      "rewards/reward_func": 0.6416638314723968,
      "step": 6050,
      "toxic_reward": 4.481846666336059
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.45,
      "epoch": 1.4319470699432892,
      "format_reward": 0.0,
      "grad_norm": 7.034083843231201,
      "image_reward": 0.2545166015625,
      "kl": 1.5230970159173012,
      "learning_rate": 5e-06,
      "loss": -0.0407,
      "reward": 0.29611208438873293,
      "reward_std": 0.7949410590808839,
      "rewards/reward_func": 0.29611208438873293,
      "step": 6060,
      "toxic_reward": 4.335282778739929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.75,
      "epoch": 1.4343100189035916,
      "format_reward": 0.0,
      "grad_norm": 3.7928450107574463,
      "image_reward": 0.24527740478515625,
      "kl": 0.8901469498872757,
      "learning_rate": 5e-06,
      "loss": -0.0367,
      "reward": 0.26578280329704285,
      "reward_std": 1.3428313750773668,
      "rewards/reward_func": 0.26578280329704285,
      "step": 6070,
      "toxic_reward": 3.388633108139038
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.625,
      "epoch": 1.436672967863894,
      "format_reward": -0.25,
      "grad_norm": 19.84122085571289,
      "image_reward": 0.2548517853021622,
      "kl": 1.0234291791915893,
      "learning_rate": 5e-06,
      "loss": 0.1725,
      "reward": 0.31002968549728394,
      "reward_std": 1.8640546321868896,
      "rewards/reward_func": 0.31002968549728394,
      "step": 6080,
      "toxic_reward": 3.8593465805053713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.6,
      "epoch": 1.4390359168241966,
      "format_reward": -0.25,
      "grad_norm": 10.626410484313965,
      "image_reward": 0.2676523834466934,
      "kl": 5.208069609105587,
      "learning_rate": 5e-06,
      "loss": 0.0794,
      "reward": 0.2428498387336731,
      "reward_std": 1.3197494292631746,
      "rewards/reward_func": 0.2428498387336731,
      "step": 6090,
      "toxic_reward": 4.562512469291687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.8,
      "epoch": 1.4413988657844992,
      "format_reward": 0.0,
      "grad_norm": 11.22333812713623,
      "image_reward": 0.2790537506341934,
      "kl": 4.089116859436035,
      "learning_rate": 5e-06,
      "loss": 0.0053,
      "reward": 1.0973919004201889,
      "reward_std": 0.9867459360510111,
      "rewards/reward_func": 1.0973919004201889,
      "step": 6100,
      "toxic_reward": 4.121850895881653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.675,
      "epoch": 1.4437618147448015,
      "format_reward": 0.0,
      "grad_norm": 2.594348907470703,
      "image_reward": 0.24527740478515625,
      "kl": 2.355099043250084,
      "learning_rate": 5e-06,
      "loss": 0.0302,
      "reward": 0.2879053592681885,
      "reward_std": 1.3369514867663383,
      "rewards/reward_func": 0.2879053592681885,
      "step": 6110,
      "toxic_reward": 3.482189404964447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.625,
      "epoch": 1.446124763705104,
      "format_reward": 0.0,
      "grad_norm": 2.740832805633545,
      "image_reward": 0.27071533203125,
      "kl": 1.8061291784048081,
      "learning_rate": 5e-06,
      "loss": 0.098,
      "reward": 0.3818982481956482,
      "reward_std": 0.8427915960550308,
      "rewards/reward_func": 0.3818982481956482,
      "step": 6120,
      "toxic_reward": 4.095608282089233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.05,
      "epoch": 1.4484877126654063,
      "format_reward": 0.0,
      "grad_norm": 16.700410842895508,
      "image_reward": 0.2543426513671875,
      "kl": 1.4419916868209839,
      "learning_rate": 5e-06,
      "loss": -0.056,
      "reward": 0.8265678405761718,
      "reward_std": 0.835081409662962,
      "rewards/reward_func": 0.8265678405761718,
      "step": 6130,
      "toxic_reward": 4.317971038818359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.75,
      "epoch": 1.4508506616257089,
      "format_reward": 0.0,
      "grad_norm": 5.467940330505371,
      "image_reward": 0.253302001953125,
      "kl": 1.128901758790016,
      "learning_rate": 5e-06,
      "loss": -0.0416,
      "reward": 0.22405808568000793,
      "reward_std": 0.430261270259507,
      "rewards/reward_func": 0.22405808568000793,
      "step": 6140,
      "toxic_reward": 4.605859112739563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.7,
      "epoch": 1.4532136105860114,
      "format_reward": 0.0,
      "grad_norm": 15.90230941772461,
      "image_reward": 0.261627197265625,
      "kl": 0.6474134013056755,
      "learning_rate": 5e-06,
      "loss": -0.0453,
      "reward": 0.23209627866744995,
      "reward_std": 0.9918515108525753,
      "rewards/reward_func": 0.23209627866744995,
      "step": 6150,
      "toxic_reward": 4.06419689655304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.85,
      "epoch": 1.455576559546314,
      "format_reward": 0.0,
      "grad_norm": 14.443000793457031,
      "image_reward": 0.2465087890625,
      "kl": 0.6866413161158562,
      "learning_rate": 5e-06,
      "loss": -0.0889,
      "reward": 0.37001847475767136,
      "reward_std": 0.7742633601650596,
      "rewards/reward_func": 0.37001847475767136,
      "step": 6160,
      "toxic_reward": 4.076632690429688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.775,
      "epoch": 1.4579395085066162,
      "format_reward": -0.25,
      "grad_norm": 16.315828323364258,
      "image_reward": 0.27055562287569046,
      "kl": 1.8430037647485733,
      "learning_rate": 5e-06,
      "loss": -0.005,
      "reward": 0.37416398525238037,
      "reward_std": 1.238390678167343,
      "rewards/reward_func": 0.37416398525238037,
      "step": 6170,
      "toxic_reward": 3.7685179471969605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.725,
      "epoch": 1.4603024574669188,
      "format_reward": -0.25,
      "grad_norm": 9.808785438537598,
      "image_reward": 0.26315511018037796,
      "kl": 3.501179130375385,
      "learning_rate": 5e-06,
      "loss": -0.0757,
      "reward": 0.4839250385761261,
      "reward_std": 1.0200102254748344,
      "rewards/reward_func": 0.4839250385761261,
      "step": 6180,
      "toxic_reward": 4.458306789398193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.1,
      "epoch": 1.462665406427221,
      "format_reward": 0.0,
      "grad_norm": 6.254196643829346,
      "image_reward": 0.2431121826171875,
      "kl": 0.6629411533474923,
      "learning_rate": 5e-06,
      "loss": -0.0243,
      "reward": 1.0291070997714997,
      "reward_std": 0.6561918726190925,
      "rewards/reward_func": 1.0291070997714997,
      "step": 6190,
      "toxic_reward": 4.277180218696595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.075,
      "epoch": 1.4650283553875236,
      "format_reward": 0.0,
      "grad_norm": 4.544678211212158,
      "image_reward": 0.288299560546875,
      "kl": 0.5533515185117721,
      "learning_rate": 5e-06,
      "loss": 0.0114,
      "reward": 0.09159567654132843,
      "reward_std": 0.6166084105148911,
      "rewards/reward_func": 0.09159567654132843,
      "step": 6200,
      "toxic_reward": 4.60030083656311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.75,
      "epoch": 1.4673913043478262,
      "format_reward": 0.0,
      "grad_norm": 2.3730123043060303,
      "image_reward": 0.2605743408203125,
      "kl": 0.9568765789270401,
      "learning_rate": 5e-06,
      "loss": 0.0904,
      "reward": 0.7965957373380661,
      "reward_std": 0.7220977865159511,
      "rewards/reward_func": 0.7965957373380661,
      "step": 6210,
      "toxic_reward": 3.7931410372257233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.05,
      "epoch": 1.4697542533081285,
      "format_reward": 0.0,
      "grad_norm": 9.046610832214355,
      "image_reward": 0.25125885009765625,
      "kl": 0.966689832508564,
      "learning_rate": 5e-06,
      "loss": 0.006,
      "reward": 0.42708381414413454,
      "reward_std": 0.8918632004410029,
      "rewards/reward_func": 0.42708381414413454,
      "step": 6220,
      "toxic_reward": 3.811506199836731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.225,
      "epoch": 1.472117202268431,
      "format_reward": -0.25,
      "grad_norm": 11.783949851989746,
      "image_reward": 0.2802093505859375,
      "kl": 1.5058857083320618,
      "learning_rate": 5e-06,
      "loss": 0.1229,
      "reward": -0.2064610540866852,
      "reward_std": 1.2439106579869985,
      "rewards/reward_func": -0.2064610540866852,
      "step": 6230,
      "toxic_reward": 3.776739251613617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.55,
      "epoch": 1.4744801512287333,
      "format_reward": -0.25,
      "grad_norm": 19.82095718383789,
      "image_reward": 0.2695220947265625,
      "kl": 4.01448056101799,
      "learning_rate": 5e-06,
      "loss": 0.0224,
      "reward": 0.12153833210468293,
      "reward_std": 1.4096519321203231,
      "rewards/reward_func": 0.12153833210468293,
      "step": 6240,
      "toxic_reward": 3.5601022720336912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.575,
      "epoch": 1.4768431001890359,
      "format_reward": 0.0,
      "grad_norm": 3.002202272415161,
      "image_reward": 0.2589111328125,
      "kl": 6.391136825084686,
      "learning_rate": 5e-06,
      "loss": 0.0246,
      "reward": 0.8984602272510529,
      "reward_std": 0.8823632273823023,
      "rewards/reward_func": 0.8984602272510529,
      "step": 6250,
      "toxic_reward": 4.186536240577698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.95,
      "epoch": 1.4792060491493384,
      "format_reward": -0.25,
      "grad_norm": 6.6309990882873535,
      "image_reward": 0.25638376772403715,
      "kl": 5.093664228916168,
      "learning_rate": 5e-06,
      "loss": -0.1168,
      "reward": 0.7955404102802277,
      "reward_std": 1.139578907750547,
      "rewards/reward_func": 0.7955404102802277,
      "step": 6260,
      "toxic_reward": 4.489496183395386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.475,
      "epoch": 1.481568998109641,
      "format_reward": -0.25,
      "grad_norm": 6.48809289932251,
      "image_reward": 0.24294535368680953,
      "kl": 2.03928547501564,
      "learning_rate": 5e-06,
      "loss": -0.0415,
      "reward": 0.3108412384986877,
      "reward_std": 1.2936087466776371,
      "rewards/reward_func": 0.3108412384986877,
      "step": 6270,
      "toxic_reward": 3.8573724269866942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.025,
      "epoch": 1.4839319470699432,
      "format_reward": 0.0,
      "grad_norm": 12.869200706481934,
      "image_reward": 0.269140625,
      "kl": 2.209321880340576,
      "learning_rate": 5e-06,
      "loss": 0.0065,
      "reward": 1.0746480822563171,
      "reward_std": 0.9140975341200829,
      "rewards/reward_func": 1.0746480822563171,
      "step": 6280,
      "toxic_reward": 4.498701477050782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.6,
      "epoch": 1.4862948960302458,
      "format_reward": -0.25,
      "grad_norm": 14.994149208068848,
      "image_reward": 0.2699289947748184,
      "kl": 1.982865560054779,
      "learning_rate": 5e-06,
      "loss": -0.0155,
      "reward": -0.04395916759967804,
      "reward_std": 1.351198247075081,
      "rewards/reward_func": -0.04395916759967804,
      "step": 6290,
      "toxic_reward": 4.100183129310608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.525,
      "epoch": 1.488657844990548,
      "format_reward": 0.0,
      "grad_norm": 13.48897647857666,
      "image_reward": 0.2738677978515625,
      "kl": 73.90320363342762,
      "learning_rate": 5e-06,
      "loss": 0.0515,
      "reward": 0.2645439743995667,
      "reward_std": 0.8113596703857183,
      "rewards/reward_func": 0.2645439743995667,
      "step": 6300,
      "toxic_reward": 4.189095830917358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.85,
      "epoch": 1.4910207939508506,
      "format_reward": -0.25,
      "grad_norm": 3.184455394744873,
      "image_reward": 0.2667582184076309,
      "kl": 1.9667763262987137,
      "learning_rate": 5e-06,
      "loss": -0.0049,
      "reward": 0.582335239648819,
      "reward_std": 1.003530977293849,
      "rewards/reward_func": 0.582335239648819,
      "step": 6310,
      "toxic_reward": 4.428107571601868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.475,
      "epoch": 1.4933837429111532,
      "format_reward": 0.0,
      "grad_norm": 16.498891830444336,
      "image_reward": 0.26070556640625,
      "kl": 2.5642897844314576,
      "learning_rate": 5e-06,
      "loss": -0.0706,
      "reward": 0.8335122138261795,
      "reward_std": 0.8066307563334704,
      "rewards/reward_func": 0.8335122138261795,
      "step": 6320,
      "toxic_reward": 3.9580556988716125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.875,
      "epoch": 1.4957466918714557,
      "format_reward": -0.25,
      "grad_norm": 10.699289321899414,
      "image_reward": 0.2347137451171875,
      "kl": 3.54896736741066,
      "learning_rate": 5e-06,
      "loss": 0.004,
      "reward": -0.006625699996948242,
      "reward_std": 1.1276695830747485,
      "rewards/reward_func": -0.006625699996948242,
      "step": 6330,
      "toxic_reward": 4.5597028732299805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.2,
      "epoch": 1.498109640831758,
      "format_reward": 0.0,
      "grad_norm": 25.74764633178711,
      "image_reward": 0.2565399169921875,
      "kl": 3.312129205465317,
      "learning_rate": 5e-06,
      "loss": -0.0109,
      "reward": 0.974502682685852,
      "reward_std": 0.8662806877866387,
      "rewards/reward_func": 0.974502682685852,
      "step": 6340,
      "toxic_reward": 4.394499397277832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.75,
      "epoch": 1.5004725897920603,
      "format_reward": -0.25,
      "grad_norm": 7.920969009399414,
      "image_reward": 0.25830586850643156,
      "kl": 4.049802941083908,
      "learning_rate": 5e-06,
      "loss": 0.1152,
      "reward": 0.04905744194984436,
      "reward_std": 1.192653514072299,
      "rewards/reward_func": 0.04905744194984436,
      "step": 6350,
      "toxic_reward": 4.2746446371078495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.45,
      "epoch": 1.5028355387523629,
      "format_reward": 0.0,
      "grad_norm": 10.33079719543457,
      "image_reward": 0.26002349853515627,
      "kl": 5.124299117922783,
      "learning_rate": 5e-06,
      "loss": 0.0575,
      "reward": 0.20074379444122314,
      "reward_std": 0.825444309413433,
      "rewards/reward_func": 0.20074379444122314,
      "step": 6360,
      "toxic_reward": 3.7550126791000364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.8,
      "epoch": 1.5051984877126654,
      "format_reward": 0.0,
      "grad_norm": 8.266423225402832,
      "image_reward": 0.262652587890625,
      "kl": 13.647933864593506,
      "learning_rate": 5e-06,
      "loss": -0.0262,
      "reward": 0.6701415419578552,
      "reward_std": 0.6827380709350109,
      "rewards/reward_func": 0.6701415419578552,
      "step": 6370,
      "toxic_reward": 4.507234740257263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.55,
      "epoch": 1.507561436672968,
      "format_reward": -0.25,
      "grad_norm": 16.820531845092773,
      "image_reward": 0.2648305267095566,
      "kl": 2943.323862874508,
      "learning_rate": 5e-06,
      "loss": 0.3168,
      "reward": -0.18676466941833497,
      "reward_std": 1.4769982114434241,
      "rewards/reward_func": -0.18676466941833497,
      "step": 6380,
      "toxic_reward": 3.6400262832641603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.85,
      "epoch": 1.5099243856332705,
      "format_reward": 0.0,
      "grad_norm": 22.587871551513672,
      "image_reward": 0.2433380126953125,
      "kl": 1.810417714715004,
      "learning_rate": 5e-06,
      "loss": 0.0137,
      "reward": 0.35521286725997925,
      "reward_std": 0.5707202635705471,
      "rewards/reward_func": 0.35521286725997925,
      "step": 6390,
      "toxic_reward": 4.4419690608978275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.85,
      "epoch": 1.5122873345935728,
      "format_reward": 0.0,
      "grad_norm": 7.745183944702148,
      "image_reward": 0.253375244140625,
      "kl": 1.7831827580928803,
      "learning_rate": 5e-06,
      "loss": 0.0178,
      "reward": -0.09840984344482422,
      "reward_std": 0.7997165352106095,
      "rewards/reward_func": -0.09840984344482422,
      "step": 6400,
      "toxic_reward": 3.674038052558899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.025,
      "epoch": 1.514650283553875,
      "format_reward": 0.0,
      "grad_norm": 6.026752948760986,
      "image_reward": 0.249884033203125,
      "kl": 4.073290675878525,
      "learning_rate": 5e-06,
      "loss": 0.0349,
      "reward": 0.7357653975486755,
      "reward_std": 1.102572639286518,
      "rewards/reward_func": 0.7357653975486755,
      "step": 6410,
      "toxic_reward": 4.154096102714538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.5,
      "epoch": 1.5170132325141776,
      "format_reward": 0.0,
      "grad_norm": 2.0699822902679443,
      "image_reward": 0.2804595947265625,
      "kl": 1.0494691252708435,
      "learning_rate": 5e-06,
      "loss": -0.0105,
      "reward": 0.3559255480766296,
      "reward_std": 0.9502544086426497,
      "rewards/reward_func": 0.3559255480766296,
      "step": 6420,
      "toxic_reward": 3.829944038391113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.725,
      "epoch": 1.5193761814744802,
      "format_reward": -0.25,
      "grad_norm": 2.6939737796783447,
      "image_reward": 0.24329833984375,
      "kl": 2.4543985188007356,
      "learning_rate": 5e-06,
      "loss": 0.0488,
      "reward": -0.22206905484199524,
      "reward_std": 1.1714405838400126,
      "rewards/reward_func": -0.22206905484199524,
      "step": 6430,
      "toxic_reward": 4.02121376991272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.8,
      "epoch": 1.5217391304347827,
      "format_reward": 0.0,
      "grad_norm": 7.930452823638916,
      "image_reward": 0.270928955078125,
      "kl": 2.3063239082694054,
      "learning_rate": 5e-06,
      "loss": 0.0354,
      "reward": -0.090572190284729,
      "reward_std": 0.8380892558023334,
      "rewards/reward_func": -0.090572190284729,
      "step": 6440,
      "toxic_reward": 4.0341674268245695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.675,
      "epoch": 1.524102079395085,
      "format_reward": -0.25,
      "grad_norm": 11.180920600891113,
      "image_reward": 0.26758829653263094,
      "kl": 1.2753668040037156,
      "learning_rate": 5e-06,
      "loss": -0.0044,
      "reward": 0.8226781934499741,
      "reward_std": 1.3871233612298965,
      "rewards/reward_func": 0.8226781934499741,
      "step": 6450,
      "toxic_reward": 3.5244659066200255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.75,
      "epoch": 1.5264650283553876,
      "format_reward": -0.25,
      "grad_norm": 21.079256057739258,
      "image_reward": 0.24825642853975297,
      "kl": 0.9987513780593872,
      "learning_rate": 5e-06,
      "loss": -0.0776,
      "reward": 0.38041144609451294,
      "reward_std": 1.5831992760300637,
      "rewards/reward_func": 0.38041144609451294,
      "step": 6460,
      "toxic_reward": 4.319032979011536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.5,
      "epoch": 1.5288279773156899,
      "format_reward": -0.25,
      "grad_norm": 6.294722557067871,
      "image_reward": 0.23506622314453124,
      "kl": 1.3047454893589019,
      "learning_rate": 5e-06,
      "loss": -0.037,
      "reward": 0.483357185125351,
      "reward_std": 1.4818070188164711,
      "rewards/reward_func": 0.483357185125351,
      "step": 6470,
      "toxic_reward": 4.28910231590271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.35,
      "epoch": 1.5311909262759924,
      "format_reward": 0.0,
      "grad_norm": 25.183557510375977,
      "image_reward": 0.2621653228998184,
      "kl": 5.334516155719757,
      "learning_rate": 5e-06,
      "loss": 0.184,
      "reward": 0.5027847826480866,
      "reward_std": 0.6526452742516995,
      "rewards/reward_func": 0.5027847826480866,
      "step": 6480,
      "toxic_reward": 4.249637746810913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.825,
      "epoch": 1.533553875236295,
      "format_reward": 0.0,
      "grad_norm": 10.60496997833252,
      "image_reward": 0.26144917905330656,
      "kl": 3.185924381017685,
      "learning_rate": 5e-06,
      "loss": -0.0162,
      "reward": 0.6137366682291031,
      "reward_std": 0.5949534647166729,
      "rewards/reward_func": 0.6137366682291031,
      "step": 6490,
      "toxic_reward": 4.377641320228577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.975,
      "epoch": 1.5359168241965975,
      "format_reward": 0.0,
      "grad_norm": 5.117782115936279,
      "image_reward": 0.2869781494140625,
      "kl": 0.5223678901791573,
      "learning_rate": 5e-06,
      "loss": 0.0356,
      "reward": 0.20925453603267669,
      "reward_std": 0.5696444906294346,
      "rewards/reward_func": 0.20925453603267669,
      "step": 6500,
      "toxic_reward": 4.292421555519104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.7,
      "epoch": 1.5382797731568998,
      "format_reward": 0.0,
      "grad_norm": 2.3820912837982178,
      "image_reward": 0.250634765625,
      "kl": 1.4595504850149155,
      "learning_rate": 5e-06,
      "loss": 0.02,
      "reward": 0.719105675816536,
      "reward_std": 0.9932258397340774,
      "rewards/reward_func": 0.719105675816536,
      "step": 6510,
      "toxic_reward": 4.147652292251587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.65,
      "epoch": 1.5406427221172023,
      "format_reward": 0.0,
      "grad_norm": 10.115738868713379,
      "image_reward": 0.2735626220703125,
      "kl": 2.502386949956417,
      "learning_rate": 5e-06,
      "loss": 0.0075,
      "reward": 0.229107666015625,
      "reward_std": 0.8675182597711683,
      "rewards/reward_func": 0.229107666015625,
      "step": 6520,
      "toxic_reward": 3.437857782840729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.975,
      "epoch": 1.5430056710775046,
      "format_reward": 0.0,
      "grad_norm": 12.080828666687012,
      "image_reward": 0.26392364501953125,
      "kl": 1.0831295281648636,
      "learning_rate": 5e-06,
      "loss": 0.0655,
      "reward": 0.5672924667596817,
      "reward_std": 1.0283904120326042,
      "rewards/reward_func": 0.5672924667596817,
      "step": 6530,
      "toxic_reward": 4.013876247406006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.825,
      "epoch": 1.5453686200378072,
      "format_reward": -0.25,
      "grad_norm": 4.358407497406006,
      "image_reward": 0.2578287750482559,
      "kl": 0.8826712548732758,
      "learning_rate": 5e-06,
      "loss": 0.0534,
      "reward": 0.1970734715461731,
      "reward_std": 1.158833772689104,
      "rewards/reward_func": 0.1970734715461731,
      "step": 6540,
      "toxic_reward": 4.346637082099915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.85,
      "epoch": 1.5477315689981097,
      "format_reward": 0.0,
      "grad_norm": 1.7384082078933716,
      "image_reward": 0.245843505859375,
      "kl": 1.1669968128204347,
      "learning_rate": 5e-06,
      "loss": 0.1047,
      "reward": 0.4790124922990799,
      "reward_std": 0.8679840985685587,
      "rewards/reward_func": 0.4790124922990799,
      "step": 6550,
      "toxic_reward": 4.215170729160309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 57.025,
      "epoch": 1.5500945179584122,
      "format_reward": -0.25,
      "grad_norm": 19.237064361572266,
      "image_reward": 0.2605519607663155,
      "kl": 22.883435368537903,
      "learning_rate": 5e-06,
      "loss": 0.0861,
      "reward": 0.3663973331451416,
      "reward_std": 1.3235621018335224,
      "rewards/reward_func": 0.3663973331451416,
      "step": 6560,
      "toxic_reward": 4.399778747558594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.5,
      "epoch": 1.5524574669187146,
      "format_reward": 0.0,
      "grad_norm": 12.169953346252441,
      "image_reward": 0.240032958984375,
      "kl": 2.908788651227951,
      "learning_rate": 5e-06,
      "loss": -0.0407,
      "reward": 0.6662415623664856,
      "reward_std": 0.9049512568861247,
      "rewards/reward_func": 0.6662415623664856,
      "step": 6570,
      "toxic_reward": 4.262545752525329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.225,
      "epoch": 1.5548204158790169,
      "format_reward": 0.0,
      "grad_norm": 14.397954940795898,
      "image_reward": 0.2709014892578125,
      "kl": 1.8459113836288452,
      "learning_rate": 5e-06,
      "loss": 0.0948,
      "reward": 0.4346597075462341,
      "reward_std": 1.024691704288125,
      "rewards/reward_func": 0.4346597075462341,
      "step": 6580,
      "toxic_reward": 3.837217903137207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.25,
      "epoch": 1.5571833648393194,
      "format_reward": 0.0,
      "grad_norm": 5.823775768280029,
      "image_reward": 0.2421417236328125,
      "kl": 361.87849075496194,
      "learning_rate": 5e-06,
      "loss": 0.1249,
      "reward": 0.7188747763633728,
      "reward_std": 0.7972227469086647,
      "rewards/reward_func": 0.7188747763633728,
      "step": 6590,
      "toxic_reward": 4.341918230056763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.025,
      "epoch": 1.559546313799622,
      "format_reward": 0.0,
      "grad_norm": 10.157111167907715,
      "image_reward": 0.2904388427734375,
      "kl": 2.8333058834075926,
      "learning_rate": 5e-06,
      "loss": -0.0125,
      "reward": 0.1938968062400818,
      "reward_std": 0.5627395014278591,
      "rewards/reward_func": 0.1938968062400818,
      "step": 6600,
      "toxic_reward": 4.045072281360627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.875,
      "epoch": 1.5619092627599245,
      "format_reward": -0.5,
      "grad_norm": 7.558686256408691,
      "image_reward": 0.28143310397863386,
      "kl": 5.3599341928958895,
      "learning_rate": 5e-06,
      "loss": -0.0333,
      "reward": 0.3363319247961044,
      "reward_std": 1.8769858199171723,
      "rewards/reward_func": 0.3363319247961044,
      "step": 6610,
      "toxic_reward": 3.4191954016685484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.3,
      "epoch": 1.5642722117202268,
      "format_reward": 0.0,
      "grad_norm": 6.938777446746826,
      "image_reward": 0.2576054885983467,
      "kl": 3.9582558915019037,
      "learning_rate": 5e-06,
      "loss": -0.0771,
      "reward": 0.785160881280899,
      "reward_std": 1.3131701787933707,
      "rewards/reward_func": 0.785160881280899,
      "step": 6620,
      "toxic_reward": 3.997890996932983
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.075,
      "epoch": 1.5666351606805293,
      "format_reward": 0.0,
      "grad_norm": 7.617331027984619,
      "image_reward": 0.2779083251953125,
      "kl": 2.7307909965515136,
      "learning_rate": 5e-06,
      "loss": 0.052,
      "reward": 0.6236707329750061,
      "reward_std": 0.8986939422786235,
      "rewards/reward_func": 0.6236707329750061,
      "step": 6630,
      "toxic_reward": 4.061939382553101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.1,
      "epoch": 1.5689981096408316,
      "format_reward": -0.5,
      "grad_norm": 11.394710540771484,
      "image_reward": 0.27598063051700594,
      "kl": 4.903402748703956,
      "learning_rate": 5e-06,
      "loss": 0.0366,
      "reward": 0.095058873295784,
      "reward_std": 1.6800902128219604,
      "rewards/reward_func": 0.095058873295784,
      "step": 6640,
      "toxic_reward": 3.813139808177948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 59.8,
      "epoch": 1.5713610586011342,
      "format_reward": 0.0,
      "grad_norm": 3.759659767150879,
      "image_reward": 0.23873443603515626,
      "kl": 7.589515461027622,
      "learning_rate": 5e-06,
      "loss": -0.0834,
      "reward": 0.44450428485870364,
      "reward_std": 0.5481153151020408,
      "rewards/reward_func": 0.44450428485870364,
      "step": 6650,
      "toxic_reward": 3.8413244128227233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.3,
      "epoch": 1.5737240075614367,
      "format_reward": 0.0,
      "grad_norm": 13.018331527709961,
      "image_reward": 0.2600331619381905,
      "kl": 3.1137637734413146,
      "learning_rate": 5e-06,
      "loss": 0.0214,
      "reward": 0.7498049587011337,
      "reward_std": 0.7316715233027935,
      "rewards/reward_func": 0.7498049587011337,
      "step": 6660,
      "toxic_reward": 4.240022134780884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.575,
      "epoch": 1.5760869565217392,
      "format_reward": 0.0,
      "grad_norm": 7.951615333557129,
      "image_reward": 0.2636627197265625,
      "kl": 0.875077161192894,
      "learning_rate": 5e-06,
      "loss": -0.1023,
      "reward": 0.6656131267547607,
      "reward_std": 0.6067664973437786,
      "rewards/reward_func": 0.6656131267547607,
      "step": 6670,
      "toxic_reward": 4.546977305412293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.725,
      "epoch": 1.5784499054820416,
      "format_reward": 0.0,
      "grad_norm": 4.168910026550293,
      "image_reward": 0.24169108122587205,
      "kl": 4.073548844456672,
      "learning_rate": 5e-06,
      "loss": -0.0498,
      "reward": 0.5430697202682495,
      "reward_std": 1.2346604462713002,
      "rewards/reward_func": 0.5430697202682495,
      "step": 6680,
      "toxic_reward": 3.895166778564453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.275,
      "epoch": 1.580812854442344,
      "format_reward": 0.0,
      "grad_norm": 2.9899535179138184,
      "image_reward": 0.23625640869140624,
      "kl": 6.596899893879891,
      "learning_rate": 5e-06,
      "loss": 0.0148,
      "reward": 0.3857423186302185,
      "reward_std": 0.7419607482850552,
      "rewards/reward_func": 0.3857423186302185,
      "step": 6690,
      "toxic_reward": 4.416726422309876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.475,
      "epoch": 1.5831758034026464,
      "format_reward": 0.0,
      "grad_norm": 17.505062103271484,
      "image_reward": 0.24977264404296876,
      "kl": 130.44692096710205,
      "learning_rate": 5e-06,
      "loss": 0.0063,
      "reward": 0.8495797365903854,
      "reward_std": 1.0700383991003037,
      "rewards/reward_func": 0.8495797365903854,
      "step": 6700,
      "toxic_reward": 3.7804057955741883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.25,
      "epoch": 1.585538752362949,
      "format_reward": -0.25,
      "grad_norm": 9.865876197814941,
      "image_reward": 0.2743357330560684,
      "kl": 1.342175406217575,
      "learning_rate": 5e-06,
      "loss": 0.0144,
      "reward": -0.01569686532020569,
      "reward_std": 1.0583332434296608,
      "rewards/reward_func": -0.01569686532020569,
      "step": 6710,
      "toxic_reward": 4.605420160293579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.65,
      "epoch": 1.5879017013232515,
      "format_reward": -0.25,
      "grad_norm": 1.4694126844406128,
      "image_reward": 0.26331074982881547,
      "kl": 1.179875871539116,
      "learning_rate": 5e-06,
      "loss": 0.0439,
      "reward": 0.19556427299976348,
      "reward_std": 1.060202201642096,
      "rewards/reward_func": 0.19556427299976348,
      "step": 6720,
      "toxic_reward": 4.195696997642517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.75,
      "epoch": 1.590264650283554,
      "format_reward": 0.0,
      "grad_norm": 14.831318855285645,
      "image_reward": 0.26193084716796877,
      "kl": 3.8945027977228164,
      "learning_rate": 5e-06,
      "loss": -0.0405,
      "reward": 0.8553763270378113,
      "reward_std": 0.7129356294870377,
      "rewards/reward_func": 0.8553763270378113,
      "step": 6730,
      "toxic_reward": 4.33803927898407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.475,
      "epoch": 1.5926275992438563,
      "format_reward": -0.25,
      "grad_norm": 19.41834831237793,
      "image_reward": 0.25118509978055953,
      "kl": 3.402638703584671,
      "learning_rate": 5e-06,
      "loss": 0.0046,
      "reward": 0.5174610838294029,
      "reward_std": 1.2720857471227647,
      "rewards/reward_func": 0.5174610838294029,
      "step": 6740,
      "toxic_reward": 3.870224565267563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.025,
      "epoch": 1.5949905482041586,
      "format_reward": 0.0,
      "grad_norm": 6.40997838973999,
      "image_reward": 0.258154296875,
      "kl": 1.6460766345262527,
      "learning_rate": 5e-06,
      "loss": 0.0986,
      "reward": 1.0767779767513275,
      "reward_std": 1.5294719189405441,
      "rewards/reward_func": 1.0767779767513275,
      "step": 6750,
      "toxic_reward": 3.531558334827423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.7,
      "epoch": 1.5973534971644612,
      "format_reward": -0.5,
      "grad_norm": 30.405113220214844,
      "image_reward": 0.25986429750919343,
      "kl": 3.0854232251644134,
      "learning_rate": 5e-06,
      "loss": -0.0433,
      "reward": 0.2282954216003418,
      "reward_std": 2.2074968218803406,
      "rewards/reward_func": 0.2282954216003418,
      "step": 6760,
      "toxic_reward": 4.195013093948364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.3,
      "epoch": 1.5997164461247637,
      "format_reward": -0.25,
      "grad_norm": 7.383143424987793,
      "image_reward": 0.26789347380399703,
      "kl": 3.230149340629578,
      "learning_rate": 5e-06,
      "loss": -0.0221,
      "reward": 0.44291332364082336,
      "reward_std": 1.4428741056472063,
      "rewards/reward_func": 0.44291332364082336,
      "step": 6770,
      "toxic_reward": 3.8504308581352236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.925,
      "epoch": 1.6020793950850662,
      "format_reward": 0.0,
      "grad_norm": 6.7697906494140625,
      "image_reward": 0.2436859130859375,
      "kl": 3.0731608659029006,
      "learning_rate": 5e-06,
      "loss": -0.0282,
      "reward": 0.9663064420223236,
      "reward_std": 0.8580235980451107,
      "rewards/reward_func": 0.9663064420223236,
      "step": 6780,
      "toxic_reward": 3.754929578304291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.725,
      "epoch": 1.6044423440453688,
      "format_reward": 0.0,
      "grad_norm": 3.5446696281433105,
      "image_reward": 0.260052490234375,
      "kl": 763.1712962627411,
      "learning_rate": 5e-06,
      "loss": 0.0939,
      "reward": 0.7344351947307587,
      "reward_std": 0.6753151521086693,
      "rewards/reward_func": 0.7344351947307587,
      "step": 6790,
      "toxic_reward": 4.139233088493347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.6,
      "epoch": 1.606805293005671,
      "format_reward": 0.0,
      "grad_norm": 21.1129093170166,
      "image_reward": 0.27585601806640625,
      "kl": 2.870541882514954,
      "learning_rate": 5e-06,
      "loss": -0.0466,
      "reward": 0.5618703544139863,
      "reward_std": 0.8203244937583805,
      "rewards/reward_func": 0.5618703544139863,
      "step": 6800,
      "toxic_reward": 4.3937297582626345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 57.125,
      "epoch": 1.6091682419659734,
      "format_reward": 0.0,
      "grad_norm": 5.104684352874756,
      "image_reward": 0.280615234375,
      "kl": 3.3239043831825255,
      "learning_rate": 5e-06,
      "loss": 0.045,
      "reward": 0.7852797448635102,
      "reward_std": 0.610455094370991,
      "rewards/reward_func": 0.7852797448635102,
      "step": 6810,
      "toxic_reward": 4.593313884735108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.725,
      "epoch": 1.611531190926276,
      "format_reward": 0.0,
      "grad_norm": 2.496898889541626,
      "image_reward": 0.2752532958984375,
      "kl": 2.6775636196136476,
      "learning_rate": 5e-06,
      "loss": -0.012,
      "reward": 0.7639135122299194,
      "reward_std": 0.9162261974066496,
      "rewards/reward_func": 0.7639135122299194,
      "step": 6820,
      "toxic_reward": 4.146224117279052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.075,
      "epoch": 1.6138941398865785,
      "format_reward": 0.0,
      "grad_norm": 6.608152866363525,
      "image_reward": 0.267572021484375,
      "kl": 3.533026337623596,
      "learning_rate": 5e-06,
      "loss": 0.0572,
      "reward": 0.44893051087856295,
      "reward_std": 0.9419144628569484,
      "rewards/reward_func": 0.44893051087856295,
      "step": 6830,
      "toxic_reward": 4.1230400681495665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.925,
      "epoch": 1.616257088846881,
      "format_reward": 0.0,
      "grad_norm": 11.28681755065918,
      "image_reward": 0.25560302734375,
      "kl": 7.369207835197448,
      "learning_rate": 5e-06,
      "loss": 0.0152,
      "reward": 0.5668485701084137,
      "reward_std": 0.749977857619524,
      "rewards/reward_func": 0.5668485701084137,
      "step": 6840,
      "toxic_reward": 4.343744564056396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.9,
      "epoch": 1.6186200378071833,
      "format_reward": -0.5,
      "grad_norm": 28.541820526123047,
      "image_reward": 0.2723876953125,
      "kl": 5.971449375152588,
      "learning_rate": 5e-06,
      "loss": -0.0574,
      "reward": -0.5758611798286438,
      "reward_std": 1.3836607769131661,
      "rewards/reward_func": -0.5758611798286438,
      "step": 6850,
      "toxic_reward": 4.467629170417785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.1,
      "epoch": 1.6209829867674859,
      "format_reward": 0.0,
      "grad_norm": 12.005922317504883,
      "image_reward": 0.2653228759765625,
      "kl": 6.38155357837677,
      "learning_rate": 5e-06,
      "loss": 0.0537,
      "reward": 0.6645367026329041,
      "reward_std": 0.6022280365228653,
      "rewards/reward_func": 0.6645367026329041,
      "step": 6860,
      "toxic_reward": 4.12990357875824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.175,
      "epoch": 1.6233459357277882,
      "format_reward": 0.0,
      "grad_norm": 4.00104284286499,
      "image_reward": 0.26532745361328125,
      "kl": 6.140709114074707,
      "learning_rate": 5e-06,
      "loss": 0.0866,
      "reward": 0.5954837799072266,
      "reward_std": 0.9484383892267942,
      "rewards/reward_func": 0.5954837799072266,
      "step": 6870,
      "toxic_reward": 4.030814599990845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.2,
      "epoch": 1.6257088846880907,
      "format_reward": 0.0,
      "grad_norm": 7.69809627532959,
      "image_reward": 0.2781280517578125,
      "kl": 8.641590279340743,
      "learning_rate": 5e-06,
      "loss": 0.0299,
      "reward": 0.35074634552001954,
      "reward_std": 0.5954089154489338,
      "rewards/reward_func": 0.35074634552001954,
      "step": 6880,
      "toxic_reward": 4.168118977546692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.6,
      "epoch": 1.6280718336483933,
      "format_reward": -0.25,
      "grad_norm": 7.882171154022217,
      "image_reward": 0.25705363005399706,
      "kl": 5.1897116780281065,
      "learning_rate": 5e-06,
      "loss": 0.0134,
      "reward": -0.19078816771507262,
      "reward_std": 1.1844907969236373,
      "rewards/reward_func": -0.19078816771507262,
      "step": 6890,
      "toxic_reward": 4.343024659156799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.925,
      "epoch": 1.6304347826086958,
      "format_reward": -0.25,
      "grad_norm": 13.869507789611816,
      "image_reward": 0.2716888427734375,
      "kl": 6.5070923328399655,
      "learning_rate": 5e-06,
      "loss": -0.0103,
      "reward": 0.15193371772766112,
      "reward_std": 1.270319462940097,
      "rewards/reward_func": 0.15193371772766112,
      "step": 6900,
      "toxic_reward": 4.229231309890747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 61.3,
      "epoch": 1.632797731568998,
      "format_reward": -0.25,
      "grad_norm": 6.519335746765137,
      "image_reward": 0.268505859375,
      "kl": 83.58075475692749,
      "learning_rate": 5e-06,
      "loss": 0.0011,
      "reward": 0.7866749823093414,
      "reward_std": 1.1198090038727968,
      "rewards/reward_func": 0.7866749823093414,
      "step": 6910,
      "toxic_reward": 4.473654842376709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.875,
      "epoch": 1.6351606805293004,
      "format_reward": 0.0,
      "grad_norm": 5.124833583831787,
      "image_reward": 0.2610076904296875,
      "kl": 2.196775460243225,
      "learning_rate": 5e-06,
      "loss": 0.015,
      "reward": 1.0072305798530579,
      "reward_std": 1.1389783814549446,
      "rewards/reward_func": 1.0072305798530579,
      "step": 6920,
      "toxic_reward": 4.3077033996582035
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.975,
      "epoch": 1.637523629489603,
      "format_reward": -0.5,
      "grad_norm": 3.5923500061035156,
      "image_reward": 0.2612147033214569,
      "kl": 4.726305472850799,
      "learning_rate": 5e-06,
      "loss": 0.0617,
      "reward": 0.24457889199256896,
      "reward_std": 1.512747337669134,
      "rewards/reward_func": 0.24457889199256896,
      "step": 6930,
      "toxic_reward": 4.234147024154663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.425,
      "epoch": 1.6398865784499055,
      "format_reward": 0.0,
      "grad_norm": 7.177937030792236,
      "image_reward": 0.2646942138671875,
      "kl": 5.225604176521301,
      "learning_rate": 5e-06,
      "loss": -0.0462,
      "reward": 0.3636160969734192,
      "reward_std": 0.6955469690263272,
      "rewards/reward_func": 0.3636160969734192,
      "step": 6940,
      "toxic_reward": 4.153347599506378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.25,
      "epoch": 1.642249527410208,
      "format_reward": 0.0,
      "grad_norm": 10.053350448608398,
      "image_reward": 0.253045654296875,
      "kl": 5.69408215880394,
      "learning_rate": 5e-06,
      "loss": 0.0027,
      "reward": 0.612578509747982,
      "reward_std": 0.6395491607487201,
      "rewards/reward_func": 0.612578509747982,
      "step": 6950,
      "toxic_reward": 3.9997507095336915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.025,
      "epoch": 1.6446124763705106,
      "format_reward": 0.0,
      "grad_norm": 43.84629440307617,
      "image_reward": 0.2462066650390625,
      "kl": 391.4977917432785,
      "learning_rate": 5e-06,
      "loss": 0.0408,
      "reward": 0.610540634393692,
      "reward_std": 1.4011766005307436,
      "rewards/reward_func": 0.610540634393692,
      "step": 6960,
      "toxic_reward": 3.513826107978821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.7,
      "epoch": 1.6469754253308129,
      "format_reward": 0.0,
      "grad_norm": 2.5391600131988525,
      "image_reward": 0.263494873046875,
      "kl": 26.704900431632996,
      "learning_rate": 5e-06,
      "loss": -0.0209,
      "reward": 0.6357394754886627,
      "reward_std": 0.9666919514536858,
      "rewards/reward_func": 0.6357394754886627,
      "step": 6970,
      "toxic_reward": 4.3967194080352785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.275,
      "epoch": 1.6493383742911152,
      "format_reward": 0.0,
      "grad_norm": 1.4004714488983154,
      "image_reward": 0.252850341796875,
      "kl": 5.729834485054016,
      "learning_rate": 5e-06,
      "loss": 0.0379,
      "reward": 0.47990578413009644,
      "reward_std": 0.5631790950894355,
      "rewards/reward_func": 0.47990578413009644,
      "step": 6980,
      "toxic_reward": 4.208314228057861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 57.5,
      "epoch": 1.6517013232514177,
      "format_reward": 0.0,
      "grad_norm": 5.806096076965332,
      "image_reward": 0.256463623046875,
      "kl": 6.840502554178238,
      "learning_rate": 5e-06,
      "loss": -0.0496,
      "reward": 0.16656889617443085,
      "reward_std": 0.9250041805207729,
      "rewards/reward_func": 0.16656889617443085,
      "step": 6990,
      "toxic_reward": 3.7445754587650297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.475,
      "epoch": 1.6540642722117203,
      "format_reward": -0.25,
      "grad_norm": 1.5651546716690063,
      "image_reward": 0.2714019775390625,
      "kl": 4.0309244930744175,
      "learning_rate": 5e-06,
      "loss": -0.0266,
      "reward": 0.3252350568771362,
      "reward_std": 1.123583555780351,
      "rewards/reward_func": 0.3252350568771362,
      "step": 7000,
      "toxic_reward": 4.437454390525818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.225,
      "epoch": 1.6564272211720228,
      "format_reward": -0.25,
      "grad_norm": 8.255953788757324,
      "image_reward": 0.2734893798828125,
      "kl": 13.629169458150864,
      "learning_rate": 5e-06,
      "loss": 0.0185,
      "reward": 0.5219172418117524,
      "reward_std": 1.4155076075345279,
      "rewards/reward_func": 0.5219172418117524,
      "step": 7010,
      "toxic_reward": 3.9341206908226014
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.525,
      "epoch": 1.658790170132325,
      "format_reward": 0.0,
      "grad_norm": 7.765683650970459,
      "image_reward": 0.2684661865234375,
      "kl": 4.484488549828529,
      "learning_rate": 5e-06,
      "loss": 0.0198,
      "reward": 0.41308672428131105,
      "reward_std": 0.6728175904601812,
      "rewards/reward_func": 0.41308672428131105,
      "step": 7020,
      "toxic_reward": 4.680417871475219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.425,
      "epoch": 1.6611531190926276,
      "format_reward": 0.0,
      "grad_norm": 6.050631523132324,
      "image_reward": 0.2446624755859375,
      "kl": 5.303134024143219,
      "learning_rate": 5e-06,
      "loss": -0.0023,
      "reward": 0.7305093944072724,
      "reward_std": 0.8056725425645709,
      "rewards/reward_func": 0.7305093944072724,
      "step": 7030,
      "toxic_reward": 4.371766519546509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.725,
      "epoch": 1.66351606805293,
      "format_reward": -0.25,
      "grad_norm": 36.67766189575195,
      "image_reward": 0.2738332122564316,
      "kl": 3.6509076714515687,
      "learning_rate": 5e-06,
      "loss": 0.0104,
      "reward": 0.1336117923259735,
      "reward_std": 1.408228962123394,
      "rewards/reward_func": 0.1336117923259735,
      "step": 7040,
      "toxic_reward": 3.868592691421509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.95,
      "epoch": 1.6658790170132325,
      "format_reward": -0.25,
      "grad_norm": 3.2863216400146484,
      "image_reward": 0.2496002197265625,
      "kl": 47.7468825340271,
      "learning_rate": 5e-06,
      "loss": -0.0431,
      "reward": -0.1912323772907257,
      "reward_std": 1.0762672819197179,
      "rewards/reward_func": -0.1912323772907257,
      "step": 7050,
      "toxic_reward": 4.286359405517578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.8,
      "epoch": 1.668241965973535,
      "format_reward": -0.75,
      "grad_norm": 15.117477416992188,
      "image_reward": 0.2343353286385536,
      "kl": 12.957087469100951,
      "learning_rate": 5e-06,
      "loss": -0.0128,
      "reward": -0.5250297307968139,
      "reward_std": 2.128708484955132,
      "rewards/reward_func": -0.5250297307968139,
      "step": 7060,
      "toxic_reward": 4.045247128605842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.15,
      "epoch": 1.6706049149338376,
      "format_reward": -0.25,
      "grad_norm": 6.236794471740723,
      "image_reward": 0.2446756988763809,
      "kl": 3.7372434973716735,
      "learning_rate": 5e-06,
      "loss": -0.0289,
      "reward": 0.1568456247448921,
      "reward_std": 1.3125899083912373,
      "rewards/reward_func": 0.1568456247448921,
      "step": 7070,
      "toxic_reward": 4.527845191955566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.0,
      "epoch": 1.6729678638941399,
      "format_reward": 0.0,
      "grad_norm": 1.8276275396347046,
      "image_reward": 0.253338623046875,
      "kl": 18.982901883125304,
      "learning_rate": 5e-06,
      "loss": -0.0591,
      "reward": 0.5796426713466645,
      "reward_std": 0.8607377586886287,
      "rewards/reward_func": 0.5796426713466645,
      "step": 7080,
      "toxic_reward": 4.365747809410095
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.0,
      "epoch": 1.6753308128544422,
      "format_reward": 0.0,
      "grad_norm": 5.115592956542969,
      "image_reward": 0.25368804931640626,
      "kl": 8.384132671356202,
      "learning_rate": 5e-06,
      "loss": -0.0358,
      "reward": 0.4894866108894348,
      "reward_std": 0.82001001983881,
      "rewards/reward_func": 0.4894866108894348,
      "step": 7090,
      "toxic_reward": 4.161544275283814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.45,
      "epoch": 1.6776937618147447,
      "format_reward": -0.25,
      "grad_norm": 4.421766757965088,
      "image_reward": 0.257010905444622,
      "kl": 1.5507995724678039,
      "learning_rate": 5e-06,
      "loss": 0.0479,
      "reward": -0.053127193450927736,
      "reward_std": 0.8562082014977932,
      "rewards/reward_func": -0.053127193450927736,
      "step": 7100,
      "toxic_reward": 4.494407868385315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.125,
      "epoch": 1.6800567107750473,
      "format_reward": 0.0,
      "grad_norm": 15.849198341369629,
      "image_reward": 0.254315185546875,
      "kl": 4.097546017169952,
      "learning_rate": 5e-06,
      "loss": -0.0412,
      "reward": 0.42282047867774963,
      "reward_std": 0.9609952576458454,
      "rewards/reward_func": 0.42282047867774963,
      "step": 7110,
      "toxic_reward": 4.278821682929992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.15,
      "epoch": 1.6824196597353498,
      "format_reward": -0.25,
      "grad_norm": 9.184070587158203,
      "image_reward": 0.2508982330560684,
      "kl": 1.9086317151784897,
      "learning_rate": 5e-06,
      "loss": -0.0676,
      "reward": -0.08927419185638427,
      "reward_std": 1.1106989961117506,
      "rewards/reward_func": -0.08927419185638427,
      "step": 7120,
      "toxic_reward": 4.535511326789856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.45,
      "epoch": 1.6847826086956523,
      "format_reward": 0.0,
      "grad_norm": 10.833540916442871,
      "image_reward": 0.2317718505859375,
      "kl": 1.5903507679700852,
      "learning_rate": 5e-06,
      "loss": -0.0026,
      "reward": 0.5086119592189788,
      "reward_std": 0.610715470276773,
      "rewards/reward_func": 0.5086119592189788,
      "step": 7130,
      "toxic_reward": 4.333575582504272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.75,
      "epoch": 1.6871455576559546,
      "format_reward": 0.0,
      "grad_norm": 5.887187480926514,
      "image_reward": 0.2499237060546875,
      "kl": 49.958091259002686,
      "learning_rate": 5e-06,
      "loss": -0.0703,
      "reward": 0.7590021967887879,
      "reward_std": 0.8256058894097805,
      "rewards/reward_func": 0.7590021967887879,
      "step": 7140,
      "toxic_reward": 4.225302958488465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.325,
      "epoch": 1.689508506616257,
      "format_reward": 0.0,
      "grad_norm": 12.265044212341309,
      "image_reward": 0.23530120849609376,
      "kl": 0.9182627111673355,
      "learning_rate": 5e-06,
      "loss": -0.0673,
      "reward": 0.3405183613300323,
      "reward_std": 0.7431152425706387,
      "rewards/reward_func": 0.3405183613300323,
      "step": 7150,
      "toxic_reward": 4.184520816802978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.175,
      "epoch": 1.6918714555765595,
      "format_reward": -0.25,
      "grad_norm": 3.966953754425049,
      "image_reward": 0.267620849609375,
      "kl": 1.64820496737957,
      "learning_rate": 5e-06,
      "loss": 0.1687,
      "reward": 0.01701483130455017,
      "reward_std": 1.4536124819889664,
      "rewards/reward_func": 0.01701483130455017,
      "step": 7160,
      "toxic_reward": 3.7502978086471557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.15,
      "epoch": 1.694234404536862,
      "format_reward": 0.0,
      "grad_norm": 5.9182000160217285,
      "image_reward": 0.25394287109375,
      "kl": 1.6780494809150697,
      "learning_rate": 5e-06,
      "loss": 0.0719,
      "reward": 0.8605155050754547,
      "reward_std": 0.9149322494864464,
      "rewards/reward_func": 0.8605155050754547,
      "step": 7170,
      "toxic_reward": 3.9694084405899046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.525,
      "epoch": 1.6965973534971646,
      "format_reward": 0.0,
      "grad_norm": 2.476659059524536,
      "image_reward": 0.272882080078125,
      "kl": 3.32854140996933,
      "learning_rate": 5e-06,
      "loss": -0.0742,
      "reward": 1.0633208215236665,
      "reward_std": 0.9789414823055267,
      "rewards/reward_func": 1.0633208215236665,
      "step": 7180,
      "toxic_reward": 4.391367101669312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.825,
      "epoch": 1.6989603024574669,
      "format_reward": 0.0,
      "grad_norm": 3.7560040950775146,
      "image_reward": 0.269110107421875,
      "kl": 1.4420736670494079,
      "learning_rate": 5e-06,
      "loss": 0.0345,
      "reward": 1.0360184490680695,
      "reward_std": 0.8136029925197363,
      "rewards/reward_func": 1.0360184490680695,
      "step": 7190,
      "toxic_reward": 3.99700380563736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.925,
      "epoch": 1.7013232514177694,
      "format_reward": 0.0,
      "grad_norm": 11.579362869262695,
      "image_reward": 0.2462371826171875,
      "kl": 0.9962957471609115,
      "learning_rate": 5e-06,
      "loss": -0.0255,
      "reward": 0.2942840725183487,
      "reward_std": 0.3486198179423809,
      "rewards/reward_func": 0.2942840725183487,
      "step": 7200,
      "toxic_reward": 3.8329622387886046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.825,
      "epoch": 1.7036862003780717,
      "format_reward": -0.5,
      "grad_norm": 15.74374008178711,
      "image_reward": 0.26438903957605364,
      "kl": 1.2382088035345078,
      "learning_rate": 5e-06,
      "loss": -0.0735,
      "reward": 0.2413632392883301,
      "reward_std": 1.651388045027852,
      "rewards/reward_func": 0.2413632392883301,
      "step": 7210,
      "toxic_reward": 4.516292905807495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.625,
      "epoch": 1.7060491493383743,
      "format_reward": 0.0,
      "grad_norm": 9.237770080566406,
      "image_reward": 0.2369354248046875,
      "kl": 1.5744222581386567,
      "learning_rate": 5e-06,
      "loss": 0.0256,
      "reward": 0.6944510787725449,
      "reward_std": 1.11760393679142,
      "rewards/reward_func": 0.6944510787725449,
      "step": 7220,
      "toxic_reward": 3.7596142530441283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.125,
      "epoch": 1.7084120982986768,
      "format_reward": 0.0,
      "grad_norm": 3.7665228843688965,
      "image_reward": 0.2577423095703125,
      "kl": 0.7259436190128327,
      "learning_rate": 5e-06,
      "loss": 0.0117,
      "reward": 0.5142745256423951,
      "reward_std": 0.6884998820722104,
      "rewards/reward_func": 0.5142745256423951,
      "step": 7230,
      "toxic_reward": 4.332102084159851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.475,
      "epoch": 1.7107750472589793,
      "format_reward": 0.0,
      "grad_norm": 4.795387268066406,
      "image_reward": 0.28794708251953127,
      "kl": 1.6049385368824005,
      "learning_rate": 5e-06,
      "loss": 0.0341,
      "reward": 0.308843332529068,
      "reward_std": 0.4225019045174122,
      "rewards/reward_func": 0.308843332529068,
      "step": 7240,
      "toxic_reward": 4.501336789131164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.0,
      "epoch": 1.7131379962192816,
      "format_reward": -0.25,
      "grad_norm": 11.164639472961426,
      "image_reward": 0.25756022036075593,
      "kl": 0.43412337452173233,
      "learning_rate": 5e-06,
      "loss": -0.026,
      "reward": 0.46165032386779786,
      "reward_std": 0.9854918915778399,
      "rewards/reward_func": 0.46165032386779786,
      "step": 7250,
      "toxic_reward": 4.23072258234024
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.925,
      "epoch": 1.715500945179584,
      "format_reward": 0.0,
      "grad_norm": 26.601303100585938,
      "image_reward": 0.24893798828125,
      "kl": 3.482639339566231,
      "learning_rate": 5e-06,
      "loss": -0.0419,
      "reward": 0.5657954633235931,
      "reward_std": 1.2434701435267925,
      "rewards/reward_func": 0.5657954633235931,
      "step": 7260,
      "toxic_reward": 4.052207565307617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.85,
      "epoch": 1.7178638941398865,
      "format_reward": -0.25,
      "grad_norm": 19.468366622924805,
      "image_reward": 0.24361775815486908,
      "kl": 0.6207199424505234,
      "learning_rate": 5e-06,
      "loss": -0.1011,
      "reward": 0.2289634108543396,
      "reward_std": 1.1521323285996914,
      "rewards/reward_func": 0.2289634108543396,
      "step": 7270,
      "toxic_reward": 4.243139553070068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.475,
      "epoch": 1.720226843100189,
      "format_reward": 0.0,
      "grad_norm": 11.427348136901855,
      "image_reward": 0.24371236115694045,
      "kl": 1.2423572808504104,
      "learning_rate": 5e-06,
      "loss": -0.0055,
      "reward": 0.44162888526916505,
      "reward_std": 1.226283924281597,
      "rewards/reward_func": 0.44162888526916505,
      "step": 7280,
      "toxic_reward": 3.9729990482330324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.725,
      "epoch": 1.7225897920604916,
      "format_reward": 0.0,
      "grad_norm": 8.307239532470703,
      "image_reward": 0.2623565673828125,
      "kl": 0.795675303786993,
      "learning_rate": 5e-06,
      "loss": 0.005,
      "reward": 0.47723318338394166,
      "reward_std": 0.5881602220237255,
      "rewards/reward_func": 0.47723318338394166,
      "step": 7290,
      "toxic_reward": 4.611540603637695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.125,
      "epoch": 1.724952741020794,
      "format_reward": 0.0,
      "grad_norm": 7.304860591888428,
      "image_reward": 0.2538177490234375,
      "kl": 1.0193208366632462,
      "learning_rate": 5e-06,
      "loss": -0.023,
      "reward": 0.17551978230476378,
      "reward_std": 0.5646818313747645,
      "rewards/reward_func": 0.17551978230476378,
      "step": 7300,
      "toxic_reward": 4.499063897132873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.275,
      "epoch": 1.7273156899810964,
      "format_reward": 0.0,
      "grad_norm": 1.351771354675293,
      "image_reward": 0.2463592529296875,
      "kl": 1.9171950757503509,
      "learning_rate": 5e-06,
      "loss": 0.0213,
      "reward": 0.466388076543808,
      "reward_std": 0.8451812721788883,
      "rewards/reward_func": 0.466388076543808,
      "step": 7310,
      "toxic_reward": 4.565359354019165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.925,
      "epoch": 1.7296786389413987,
      "format_reward": -0.25,
      "grad_norm": 2.364166021347046,
      "image_reward": 0.25976969450712206,
      "kl": 0.5259292095899581,
      "learning_rate": 5e-06,
      "loss": -0.084,
      "reward": 0.45669102370738984,
      "reward_std": 1.098591622710228,
      "rewards/reward_func": 0.45669102370738984,
      "step": 7320,
      "toxic_reward": 4.627902317047119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.825,
      "epoch": 1.7320415879017013,
      "format_reward": 0.0,
      "grad_norm": 23.96133804321289,
      "image_reward": 0.2389556884765625,
      "kl": 1.1734901428222657,
      "learning_rate": 5e-06,
      "loss": 0.038,
      "reward": 0.7277517914772034,
      "reward_std": 0.8356013357639313,
      "rewards/reward_func": 0.7277517914772034,
      "step": 7330,
      "toxic_reward": 4.407384157180786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.0,
      "epoch": 1.7344045368620038,
      "format_reward": 0.0,
      "grad_norm": 17.774612426757812,
      "image_reward": 0.2680206298828125,
      "kl": 4.040656617283821,
      "learning_rate": 5e-06,
      "loss": 0.0436,
      "reward": 0.2283779501914978,
      "reward_std": 0.34994165217503903,
      "rewards/reward_func": 0.2283779501914978,
      "step": 7340,
      "toxic_reward": 4.637366437911988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.25,
      "epoch": 1.7367674858223063,
      "format_reward": -0.25,
      "grad_norm": 12.662446022033691,
      "image_reward": 0.24230550229549408,
      "kl": 0.5235348105430603,
      "learning_rate": 5e-06,
      "loss": -0.042,
      "reward": 0.8077804684638977,
      "reward_std": 1.315062115341425,
      "rewards/reward_func": 0.8077804684638977,
      "step": 7350,
      "toxic_reward": 4.6036452293396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.75,
      "epoch": 1.7391304347826086,
      "format_reward": 0.0,
      "grad_norm": 2.7947723865509033,
      "image_reward": 0.259271240234375,
      "kl": 0.5023418068885803,
      "learning_rate": 5e-06,
      "loss": -0.0101,
      "reward": 0.47644210457801817,
      "reward_std": 0.6371240261942148,
      "rewards/reward_func": 0.47644210457801817,
      "step": 7360,
      "toxic_reward": 4.352305841445923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.975,
      "epoch": 1.7414933837429112,
      "format_reward": 0.0,
      "grad_norm": 6.967306137084961,
      "image_reward": 0.2540252685546875,
      "kl": 0.5360975474119186,
      "learning_rate": 5e-06,
      "loss": 0.0708,
      "reward": 0.5753240287303925,
      "reward_std": 0.8622719066217541,
      "rewards/reward_func": 0.5753240287303925,
      "step": 7370,
      "toxic_reward": 4.0306689739227295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.75,
      "epoch": 1.7438563327032135,
      "format_reward": -0.5,
      "grad_norm": 31.72753143310547,
      "image_reward": 0.22639973908662797,
      "kl": 0.5255977511405945,
      "learning_rate": 5e-06,
      "loss": 0.0039,
      "reward": -0.24822215884923934,
      "reward_std": 1.6855425260961057,
      "rewards/reward_func": -0.24822215884923934,
      "step": 7380,
      "toxic_reward": 3.752596640586853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.375,
      "epoch": 1.746219281663516,
      "format_reward": 0.0,
      "grad_norm": 7.6846818923950195,
      "image_reward": 0.2564056396484375,
      "kl": 3.591386225819588,
      "learning_rate": 5e-06,
      "loss": 0.0212,
      "reward": 0.12304354310035706,
      "reward_std": 0.8115306086838245,
      "rewards/reward_func": 0.12304354310035706,
      "step": 7390,
      "toxic_reward": 3.613353615999222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.65,
      "epoch": 1.7485822306238186,
      "format_reward": 0.0,
      "grad_norm": 2.726175308227539,
      "image_reward": 0.283404541015625,
      "kl": 0.9659576997160911,
      "learning_rate": 5e-06,
      "loss": 0.0252,
      "reward": 0.3961315780878067,
      "reward_std": 1.0492550559341907,
      "rewards/reward_func": 0.3961315780878067,
      "step": 7400,
      "toxic_reward": 3.501691198348999
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.95,
      "epoch": 1.750945179584121,
      "format_reward": 0.0,
      "grad_norm": 3.0125391483306885,
      "image_reward": 0.2607086181640625,
      "kl": 0.6532519310712814,
      "learning_rate": 5e-06,
      "loss": 0.0276,
      "reward": 0.4769218623638153,
      "reward_std": 0.6247519843280316,
      "rewards/reward_func": 0.4769218623638153,
      "step": 7410,
      "toxic_reward": 4.560657954216003
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.425,
      "epoch": 1.7533081285444234,
      "format_reward": 0.0,
      "grad_norm": 18.774812698364258,
      "image_reward": 0.24655609130859374,
      "kl": 1.901971572637558,
      "learning_rate": 5e-06,
      "loss": 0.0145,
      "reward": 0.6345466494560241,
      "reward_std": 1.1331901341676711,
      "rewards/reward_func": 0.6345466494560241,
      "step": 7420,
      "toxic_reward": 4.449591946601868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.375,
      "epoch": 1.755671077504726,
      "format_reward": 0.0,
      "grad_norm": 4.1103057861328125,
      "image_reward": 0.265411376953125,
      "kl": 1.7676091372966767,
      "learning_rate": 5e-06,
      "loss": 0.0034,
      "reward": 0.6921305894851685,
      "reward_std": 0.6238477535545826,
      "rewards/reward_func": 0.6921305894851685,
      "step": 7430,
      "toxic_reward": 3.859569197893143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.65,
      "epoch": 1.7580340264650283,
      "format_reward": 0.0,
      "grad_norm": 2.0048232078552246,
      "image_reward": 0.24298477172851562,
      "kl": 4.202221667766571,
      "learning_rate": 5e-06,
      "loss": 0.1367,
      "reward": 0.9155125916004181,
      "reward_std": 0.7328770853579044,
      "rewards/reward_func": 0.9155125916004181,
      "step": 7440,
      "toxic_reward": 4.531428098678589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.25,
      "epoch": 1.7603969754253308,
      "format_reward": 0.0,
      "grad_norm": 20.737003326416016,
      "image_reward": 0.25689697265625,
      "kl": 16.54909121990204,
      "learning_rate": 5e-06,
      "loss": 0.037,
      "reward": 0.8588055372238159,
      "reward_std": 0.9005012600682676,
      "rewards/reward_func": 0.8588055372238159,
      "step": 7450,
      "toxic_reward": 4.513736462593078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.7,
      "epoch": 1.7627599243856333,
      "format_reward": 0.0,
      "grad_norm": 1.5940968990325928,
      "image_reward": 0.26309814453125,
      "kl": 2.317558985948563,
      "learning_rate": 5e-06,
      "loss": 0.0018,
      "reward": 0.20084644556045533,
      "reward_std": 0.7237232834100723,
      "rewards/reward_func": 0.20084644556045533,
      "step": 7460,
      "toxic_reward": 4.334891009330749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.325,
      "epoch": 1.7651228733459359,
      "format_reward": 0.0,
      "grad_norm": 10.07941722869873,
      "image_reward": 0.2528594970703125,
      "kl": 1.3212820410728454,
      "learning_rate": 5e-06,
      "loss": -0.0444,
      "reward": 1.2387877494096755,
      "reward_std": 0.8179315060377121,
      "rewards/reward_func": 1.2387877494096755,
      "step": 7470,
      "toxic_reward": 4.363593196868896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.825,
      "epoch": 1.7674858223062382,
      "format_reward": 0.0,
      "grad_norm": 28.392396926879883,
      "image_reward": 0.25749053955078127,
      "kl": 2.198029878735542,
      "learning_rate": 5e-06,
      "loss": 0.0322,
      "reward": 0.1901194632053375,
      "reward_std": 0.5339192871004343,
      "rewards/reward_func": 0.1901194632053375,
      "step": 7480,
      "toxic_reward": 4.514597225189209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.7,
      "epoch": 1.7698487712665405,
      "format_reward": 0.0,
      "grad_norm": 26.77941131591797,
      "image_reward": 0.241033935546875,
      "kl": 6.588536351919174,
      "learning_rate": 5e-06,
      "loss": 0.0218,
      "reward": 0.2174743801355362,
      "reward_std": 0.8413432762026787,
      "rewards/reward_func": 0.2174743801355362,
      "step": 7490,
      "toxic_reward": 4.284235906600952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.55,
      "epoch": 1.772211720226843,
      "format_reward": -0.25,
      "grad_norm": 18.408794403076172,
      "image_reward": 0.2581207275390625,
      "kl": 3.106099420785904,
      "learning_rate": 5e-06,
      "loss": 0.0257,
      "reward": 0.31152122020721434,
      "reward_std": 1.2936958684585989,
      "rewards/reward_func": 0.31152122020721434,
      "step": 7500,
      "toxic_reward": 4.310132288932801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.475,
      "epoch": 1.7745746691871456,
      "format_reward": 0.0,
      "grad_norm": 32.10823440551758,
      "image_reward": 0.232781982421875,
      "kl": 11.768871355056763,
      "learning_rate": 5e-06,
      "loss": -0.0201,
      "reward": 1.5193881750106812,
      "reward_std": 0.8748866233974695,
      "rewards/reward_func": 1.5193881750106812,
      "step": 7510,
      "toxic_reward": 4.612711477279663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.575,
      "epoch": 1.776937618147448,
      "format_reward": 0.0,
      "grad_norm": 10.912269592285156,
      "image_reward": 0.235693359375,
      "kl": 2.0526355147361754,
      "learning_rate": 5e-06,
      "loss": 0.1284,
      "reward": 1.3539286196231841,
      "reward_std": 0.9052736334502697,
      "rewards/reward_func": 1.3539286196231841,
      "step": 7520,
      "toxic_reward": 4.487947154045105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.575,
      "epoch": 1.7793005671077504,
      "format_reward": 0.0,
      "grad_norm": 13.928491592407227,
      "image_reward": 0.2556488037109375,
      "kl": 21.833010697364806,
      "learning_rate": 5e-06,
      "loss": -0.1174,
      "reward": 0.5344179272651672,
      "reward_std": 0.7245766028761864,
      "rewards/reward_func": 0.5344179272651672,
      "step": 7530,
      "toxic_reward": 4.373207831382752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.975,
      "epoch": 1.781663516068053,
      "format_reward": 0.0,
      "grad_norm": 8.675307273864746,
      "image_reward": 0.24088897705078124,
      "kl": 1.3107800006866455,
      "learning_rate": 5e-06,
      "loss": 0.0403,
      "reward": 0.04000200629234314,
      "reward_std": 1.0572677969932556,
      "rewards/reward_func": 0.04000200629234314,
      "step": 7540,
      "toxic_reward": 4.048869323730469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.975,
      "epoch": 1.7840264650283553,
      "format_reward": -0.25,
      "grad_norm": 3.656561851501465,
      "image_reward": 0.24349263608455657,
      "kl": 3.6083962321281433,
      "learning_rate": 5e-06,
      "loss": -0.0326,
      "reward": 0.10396124720573426,
      "reward_std": 1.0819443106651305,
      "rewards/reward_func": 0.10396124720573426,
      "step": 7550,
      "toxic_reward": 4.45411868095398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.6,
      "epoch": 1.7863894139886578,
      "format_reward": 0.0,
      "grad_norm": 2.0053718090057373,
      "image_reward": 0.280938720703125,
      "kl": 1.8616322338581086,
      "learning_rate": 5e-06,
      "loss": -0.0035,
      "reward": 0.602351513504982,
      "reward_std": 0.8774395015090704,
      "rewards/reward_func": 0.602351513504982,
      "step": 7560,
      "toxic_reward": 3.8221758723258974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.15,
      "epoch": 1.7887523629489603,
      "format_reward": 0.0,
      "grad_norm": 6.999305248260498,
      "image_reward": 0.24803619384765624,
      "kl": 1.7729626595973969,
      "learning_rate": 5e-06,
      "loss": -0.004,
      "reward": 0.33846797943115237,
      "reward_std": 0.587756198644638,
      "rewards/reward_func": 0.33846797943115237,
      "step": 7570,
      "toxic_reward": 4.2568159103393555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.375,
      "epoch": 1.7911153119092629,
      "format_reward": 0.0,
      "grad_norm": 12.467576026916504,
      "image_reward": 0.2431640625,
      "kl": 1.3180940926074982,
      "learning_rate": 5e-06,
      "loss": 0.0225,
      "reward": 0.6568324744701386,
      "reward_std": 0.5710492163896561,
      "rewards/reward_func": 0.6568324744701386,
      "step": 7580,
      "toxic_reward": 4.575870084762573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.675,
      "epoch": 1.7934782608695652,
      "format_reward": 0.0,
      "grad_norm": 41.636165618896484,
      "image_reward": 0.2553070068359375,
      "kl": 1.2196908950805665,
      "learning_rate": 5e-06,
      "loss": 0.0276,
      "reward": 0.9933471500873565,
      "reward_std": 0.8478576868772507,
      "rewards/reward_func": 0.9933471500873565,
      "step": 7590,
      "toxic_reward": 4.177789008617401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.3,
      "epoch": 1.7958412098298677,
      "format_reward": 0.0,
      "grad_norm": 8.115588188171387,
      "image_reward": 0.27226715087890624,
      "kl": 5.791901814937591,
      "learning_rate": 5e-06,
      "loss": 0.0022,
      "reward": 0.3163196682929993,
      "reward_std": 0.8629786409437656,
      "rewards/reward_func": 0.3163196682929993,
      "step": 7600,
      "toxic_reward": 3.73489425778389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.95,
      "epoch": 1.79820415879017,
      "format_reward": 0.0,
      "grad_norm": 2.3679516315460205,
      "image_reward": 0.2376190185546875,
      "kl": 4.311083900928497,
      "learning_rate": 5e-06,
      "loss": -0.0175,
      "reward": 0.6290358543395996,
      "reward_std": 1.0244077319279312,
      "rewards/reward_func": 0.6290358543395996,
      "step": 7610,
      "toxic_reward": 4.054656505584717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.35,
      "epoch": 1.8005671077504726,
      "format_reward": 0.0,
      "grad_norm": 2.1850380897521973,
      "image_reward": 0.247137451171875,
      "kl": 3.278796100616455,
      "learning_rate": 5e-06,
      "loss": 0.056,
      "reward": 1.2004601210355759,
      "reward_std": 0.7055684822611511,
      "rewards/reward_func": 1.2004601210355759,
      "step": 7620,
      "toxic_reward": 3.5256235122680666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.75,
      "epoch": 1.802930056710775,
      "format_reward": -0.25,
      "grad_norm": 3.8605425357818604,
      "image_reward": 0.24492238312959672,
      "kl": 7.6126263558864595,
      "learning_rate": 5e-06,
      "loss": -0.0959,
      "reward": 0.37777516841888426,
      "reward_std": 1.1775035494938493,
      "rewards/reward_func": 0.37777516841888426,
      "step": 7630,
      "toxic_reward": 4.522587513923645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.375,
      "epoch": 1.8052930056710776,
      "format_reward": -0.25,
      "grad_norm": 6.144404411315918,
      "image_reward": 0.241168212890625,
      "kl": 1.456436914205551,
      "learning_rate": 5e-06,
      "loss": -0.0231,
      "reward": 0.36865578293800355,
      "reward_std": 1.7164668783545494,
      "rewards/reward_func": 0.36865578293800355,
      "step": 7640,
      "toxic_reward": 3.9745461702346803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.775,
      "epoch": 1.80765595463138,
      "format_reward": 0.0,
      "grad_norm": 33.95363998413086,
      "image_reward": 0.2472625732421875,
      "kl": 2.2694355845451355,
      "learning_rate": 5e-06,
      "loss": -0.1037,
      "reward": 0.8588967323303223,
      "reward_std": 1.019287913478911,
      "rewards/reward_func": 0.8588967323303223,
      "step": 7650,
      "toxic_reward": 4.213714742660523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.35,
      "epoch": 1.8100189035916823,
      "format_reward": -0.25,
      "grad_norm": 6.865695953369141,
      "image_reward": 0.25559844970703127,
      "kl": 6.3857537567615505,
      "learning_rate": 5e-06,
      "loss": -0.0924,
      "reward": -0.07846069931983948,
      "reward_std": 1.1336833463981748,
      "rewards/reward_func": -0.07846069931983948,
      "step": 7660,
      "toxic_reward": 4.2933889627456665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.825,
      "epoch": 1.8123818525519848,
      "format_reward": 0.0,
      "grad_norm": 2.160090923309326,
      "image_reward": 0.2841064453125,
      "kl": 5.202520692348481,
      "learning_rate": 5e-06,
      "loss": 0.006,
      "reward": 1.153634887933731,
      "reward_std": 1.2888424217700958,
      "rewards/reward_func": 1.153634887933731,
      "step": 7670,
      "toxic_reward": 3.994613242149353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.975,
      "epoch": 1.8147448015122873,
      "format_reward": 0.0,
      "grad_norm": 3.903553009033203,
      "image_reward": 0.26880950927734376,
      "kl": 63.095464119315146,
      "learning_rate": 5e-06,
      "loss": 0.001,
      "reward": 1.0250155806541443,
      "reward_std": 0.7393251709640026,
      "rewards/reward_func": 1.0250155806541443,
      "step": 7680,
      "toxic_reward": 3.694820535182953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.15,
      "epoch": 1.8171077504725899,
      "format_reward": 0.0,
      "grad_norm": 15.986948013305664,
      "image_reward": 0.2573333740234375,
      "kl": 3.8690971970558166,
      "learning_rate": 5e-06,
      "loss": 0.0365,
      "reward": 0.9120604813098907,
      "reward_std": 0.8725108332931996,
      "rewards/reward_func": 0.9120604813098907,
      "step": 7690,
      "toxic_reward": 4.068342316150665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.65,
      "epoch": 1.8194706994328924,
      "format_reward": 0.0,
      "grad_norm": 2.521322727203369,
      "image_reward": 0.232586669921875,
      "kl": 1.3404993683099746,
      "learning_rate": 5e-06,
      "loss": -0.0105,
      "reward": 0.08191419243812562,
      "reward_std": 0.6063120868057013,
      "rewards/reward_func": 0.08191419243812562,
      "step": 7700,
      "toxic_reward": 4.285334658622742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.65,
      "epoch": 1.8218336483931947,
      "format_reward": 0.0,
      "grad_norm": 10.563508033752441,
      "image_reward": 0.251318359375,
      "kl": 4.375722473859787,
      "learning_rate": 5e-06,
      "loss": 0.0574,
      "reward": 0.7043181240558625,
      "reward_std": 0.5366579249501229,
      "rewards/reward_func": 0.7043181240558625,
      "step": 7710,
      "toxic_reward": 4.468820595741272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.175,
      "epoch": 1.824196597353497,
      "format_reward": -0.25,
      "grad_norm": 5.306228160858154,
      "image_reward": 0.25533854216337204,
      "kl": 1.2862511157989502,
      "learning_rate": 5e-06,
      "loss": 0.0064,
      "reward": 0.2715910017490387,
      "reward_std": 1.3802445553243161,
      "rewards/reward_func": 0.2715910017490387,
      "step": 7720,
      "toxic_reward": 4.128815948963165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.525,
      "epoch": 1.8265595463137996,
      "format_reward": 0.0,
      "grad_norm": 4.18682336807251,
      "image_reward": 0.22406005859375,
      "kl": 13.262214809656143,
      "learning_rate": 5e-06,
      "loss": -0.0483,
      "reward": 0.43178263306617737,
      "reward_std": 0.5340902636758983,
      "rewards/reward_func": 0.43178263306617737,
      "step": 7730,
      "toxic_reward": 4.167550274729729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.15,
      "epoch": 1.8289224952741021,
      "format_reward": 0.0,
      "grad_norm": 16.534120559692383,
      "image_reward": 0.2627288818359375,
      "kl": 4.888235807418823,
      "learning_rate": 5e-06,
      "loss": 0.0231,
      "reward": 0.46792620718479155,
      "reward_std": 0.6471607919782401,
      "rewards/reward_func": 0.46792620718479155,
      "step": 7740,
      "toxic_reward": 4.068465518951416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.225,
      "epoch": 1.8312854442344046,
      "format_reward": 0.0,
      "grad_norm": 10.179228782653809,
      "image_reward": 0.249444580078125,
      "kl": 3.951664477586746,
      "learning_rate": 5e-06,
      "loss": -0.0106,
      "reward": 1.0039419054985046,
      "reward_std": 0.8490265306085348,
      "rewards/reward_func": 1.0039419054985046,
      "step": 7750,
      "toxic_reward": 4.28922358751297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.0,
      "epoch": 1.833648393194707,
      "format_reward": 0.0,
      "grad_norm": 1.3173015117645264,
      "image_reward": 0.2619903564453125,
      "kl": 3.221765196323395,
      "learning_rate": 5e-06,
      "loss": -0.0057,
      "reward": 0.5499142289161683,
      "reward_std": 0.8114865634590387,
      "rewards/reward_func": 0.5499142289161683,
      "step": 7760,
      "toxic_reward": 4.202396821975708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.95,
      "epoch": 1.8360113421550095,
      "format_reward": -0.25,
      "grad_norm": 9.477835655212402,
      "image_reward": 0.275567626953125,
      "kl": 7.138307851552963,
      "learning_rate": 5e-06,
      "loss": -0.0182,
      "reward": 0.5576439201831818,
      "reward_std": 1.718572654016316,
      "rewards/reward_func": 0.5576439201831818,
      "step": 7770,
      "toxic_reward": 3.892837381362915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.325,
      "epoch": 1.8383742911153118,
      "format_reward": -0.25,
      "grad_norm": 15.364556312561035,
      "image_reward": 0.22845306396484374,
      "kl": 5.752876976132393,
      "learning_rate": 5e-06,
      "loss": 0.0724,
      "reward": 0.4619426131248474,
      "reward_std": 0.994839246571064,
      "rewards/reward_func": 0.4619426131248474,
      "step": 7780,
      "toxic_reward": 4.777106142044067
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 62.75,
      "epoch": 1.8407372400756143,
      "format_reward": -0.25,
      "grad_norm": 13.07127571105957,
      "image_reward": 0.24772542268037795,
      "kl": 2.2956355273723603,
      "learning_rate": 5e-06,
      "loss": -0.0286,
      "reward": 0.23627470731735228,
      "reward_std": 1.3513500357046724,
      "rewards/reward_func": 0.23627470731735228,
      "step": 7790,
      "toxic_reward": 3.998799777030945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.675,
      "epoch": 1.8431001890359169,
      "format_reward": -0.25,
      "grad_norm": 1.4357541799545288,
      "image_reward": 0.2586761474609375,
      "kl": 2.018620651960373,
      "learning_rate": 5e-06,
      "loss": 0.0136,
      "reward": 0.5957891523838044,
      "reward_std": 1.3981972932815552,
      "rewards/reward_func": 0.5957891523838044,
      "step": 7800,
      "toxic_reward": 3.74977787733078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.9,
      "epoch": 1.8454631379962194,
      "format_reward": 0.0,
      "grad_norm": 12.382879257202148,
      "image_reward": 0.251861572265625,
      "kl": 2.0946659803390504,
      "learning_rate": 5e-06,
      "loss": -0.0109,
      "reward": 0.2852811634540558,
      "reward_std": 0.7155913963913918,
      "rewards/reward_func": 0.2852811634540558,
      "step": 7810,
      "toxic_reward": 4.4501420021057125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.125,
      "epoch": 1.8478260869565217,
      "format_reward": 0.0,
      "grad_norm": 4.438508987426758,
      "image_reward": 0.270166015625,
      "kl": 1.658120059967041,
      "learning_rate": 5e-06,
      "loss": 0.0217,
      "reward": 0.8978000760078431,
      "reward_std": 1.2586904138326644,
      "rewards/reward_func": 0.8978000760078431,
      "step": 7820,
      "toxic_reward": 4.126551675796509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.9,
      "epoch": 1.850189035916824,
      "format_reward": 0.0,
      "grad_norm": 1.4302005767822266,
      "image_reward": 0.2738861083984375,
      "kl": 1.6736175537109375,
      "learning_rate": 5e-06,
      "loss": -0.1503,
      "reward": 0.2234538435935974,
      "reward_std": 0.7356585245579481,
      "rewards/reward_func": 0.2234538435935974,
      "step": 7830,
      "toxic_reward": 3.9116656303405763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.125,
      "epoch": 1.8525519848771266,
      "format_reward": 0.0,
      "grad_norm": 5.846213340759277,
      "image_reward": 0.2658111572265625,
      "kl": 1.36658373773098,
      "learning_rate": 5e-06,
      "loss": -0.0164,
      "reward": -0.04418985247611999,
      "reward_std": 0.827529611485079,
      "rewards/reward_func": -0.04418985247611999,
      "step": 7840,
      "toxic_reward": 3.892432355880737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.475,
      "epoch": 1.8549149338374291,
      "format_reward": 0.0,
      "grad_norm": 5.060561656951904,
      "image_reward": 0.23461151123046875,
      "kl": 0.5276786342263222,
      "learning_rate": 5e-06,
      "loss": -0.003,
      "reward": 0.7852385342121124,
      "reward_std": 0.9399228170514107,
      "rewards/reward_func": 0.7852385342121124,
      "step": 7850,
      "toxic_reward": 3.641209203004837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.025,
      "epoch": 1.8572778827977316,
      "format_reward": 0.0,
      "grad_norm": 2.505263566970825,
      "image_reward": 0.259368896484375,
      "kl": 1.0133032470941543,
      "learning_rate": 5e-06,
      "loss": 0.0581,
      "reward": 0.8989585757255554,
      "reward_std": 0.917613423243165,
      "rewards/reward_func": 0.8989585757255554,
      "step": 7860,
      "toxic_reward": 4.13404905796051
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.775,
      "epoch": 1.8596408317580342,
      "format_reward": 0.0,
      "grad_norm": 37.166500091552734,
      "image_reward": 0.25547637939453127,
      "kl": 2.1411855638027193,
      "learning_rate": 5e-06,
      "loss": 0.1396,
      "reward": 0.21031073927879335,
      "reward_std": 0.6978237416595221,
      "rewards/reward_func": 0.21031073927879335,
      "step": 7870,
      "toxic_reward": 4.086808681488037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.05,
      "epoch": 1.8620037807183365,
      "format_reward": 0.0,
      "grad_norm": 1.7183008193969727,
      "image_reward": 0.2409576416015625,
      "kl": 0.6873624622821808,
      "learning_rate": 5e-06,
      "loss": 0.033,
      "reward": 0.7567365884780883,
      "reward_std": 0.95932078063488,
      "rewards/reward_func": 0.7567365884780883,
      "step": 7880,
      "toxic_reward": 4.077835154533386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.95,
      "epoch": 1.8643667296786388,
      "format_reward": 0.0,
      "grad_norm": 1.375571846961975,
      "image_reward": 0.251434326171875,
      "kl": 1.8803806602954865,
      "learning_rate": 5e-06,
      "loss": -0.0674,
      "reward": 0.1468454658985138,
      "reward_std": 0.7339655995368958,
      "rewards/reward_func": 0.1468454658985138,
      "step": 7890,
      "toxic_reward": 4.2400289416313175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.625,
      "epoch": 1.8667296786389413,
      "format_reward": 0.0,
      "grad_norm": 2.778831720352173,
      "image_reward": 0.273419189453125,
      "kl": 12.759307652711868,
      "learning_rate": 5e-06,
      "loss": 0.091,
      "reward": 0.2764736473560333,
      "reward_std": 0.6703889116644859,
      "rewards/reward_func": 0.2764736473560333,
      "step": 7900,
      "toxic_reward": 4.633634448051453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.525,
      "epoch": 1.8690926275992439,
      "format_reward": 0.0,
      "grad_norm": 14.088724136352539,
      "image_reward": 0.23843803405761718,
      "kl": 2.752323019504547,
      "learning_rate": 5e-06,
      "loss": -0.0523,
      "reward": 0.44507230520248414,
      "reward_std": 0.8451843298971653,
      "rewards/reward_func": 0.44507230520248414,
      "step": 7910,
      "toxic_reward": 3.7819975137710573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.275,
      "epoch": 1.8714555765595464,
      "format_reward": 0.0,
      "grad_norm": 12.696130752563477,
      "image_reward": 0.25413665771484373,
      "kl": 2.022602713108063,
      "learning_rate": 5e-06,
      "loss": 0.0137,
      "reward": 0.6168730854988098,
      "reward_std": 1.198334063589573,
      "rewards/reward_func": 0.6168730854988098,
      "step": 7920,
      "toxic_reward": 3.9178677558898927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.75,
      "epoch": 1.8738185255198487,
      "format_reward": 0.0,
      "grad_norm": 6.441836357116699,
      "image_reward": 0.23479461669921875,
      "kl": 2.0815513670444488,
      "learning_rate": 5e-06,
      "loss": 0.099,
      "reward": 0.49921011328697207,
      "reward_std": 0.8878588248044252,
      "rewards/reward_func": 0.49921011328697207,
      "step": 7930,
      "toxic_reward": 4.21897873878479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.575,
      "epoch": 1.8761814744801513,
      "format_reward": -0.25,
      "grad_norm": 17.865110397338867,
      "image_reward": 0.2600982666015625,
      "kl": 2.150428944826126,
      "learning_rate": 5e-06,
      "loss": -0.0837,
      "reward": 0.48849809169769287,
      "reward_std": 1.4677658422850073,
      "rewards/reward_func": 0.48849809169769287,
      "step": 7940,
      "toxic_reward": 4.1003117799758915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.925,
      "epoch": 1.8785444234404536,
      "format_reward": 0.0,
      "grad_norm": 6.697957992553711,
      "image_reward": 0.25945892333984377,
      "kl": 2.223458543419838,
      "learning_rate": 5e-06,
      "loss": 0.0562,
      "reward": 0.20426468104124068,
      "reward_std": 0.5012361383065581,
      "rewards/reward_func": 0.20426468104124068,
      "step": 7950,
      "toxic_reward": 4.173866260051727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.075,
      "epoch": 1.8809073724007561,
      "format_reward": 0.0,
      "grad_norm": 5.58077335357666,
      "image_reward": 0.25069351196289064,
      "kl": 8.21664493083954,
      "learning_rate": 5e-06,
      "loss": -0.0792,
      "reward": 0.7380830064415932,
      "reward_std": 1.2196707382798195,
      "rewards/reward_func": 0.7380830064415932,
      "step": 7960,
      "toxic_reward": 3.5894944429397584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.475,
      "epoch": 1.8832703213610587,
      "format_reward": 0.0,
      "grad_norm": 8.384510040283203,
      "image_reward": 0.23923797607421876,
      "kl": 21.914335840940474,
      "learning_rate": 5e-06,
      "loss": -0.0204,
      "reward": 0.8768561869859696,
      "reward_std": 0.7653445459902286,
      "rewards/reward_func": 0.8768561869859696,
      "step": 7970,
      "toxic_reward": 3.7090541243553163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.075,
      "epoch": 1.8856332703213612,
      "format_reward": -0.25,
      "grad_norm": 3.9271442890167236,
      "image_reward": 0.2266026809811592,
      "kl": 1.7329542875289916,
      "learning_rate": 5e-06,
      "loss": 0.0735,
      "reward": 0.006925755739212036,
      "reward_std": 1.2168598100543022,
      "rewards/reward_func": 0.006925755739212036,
      "step": 7980,
      "toxic_reward": 4.598031067848206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.375,
      "epoch": 1.8879962192816635,
      "format_reward": 0.0,
      "grad_norm": 17.941791534423828,
      "image_reward": 0.24617818146944045,
      "kl": 3.56347342133522,
      "learning_rate": 5e-06,
      "loss": -0.1121,
      "reward": 0.15787817239761354,
      "reward_std": 0.5696724381297826,
      "rewards/reward_func": 0.15787817239761354,
      "step": 7990,
      "toxic_reward": 4.376770114898681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.675,
      "epoch": 1.8903591682419658,
      "format_reward": 0.0,
      "grad_norm": 26.69174575805664,
      "image_reward": 0.2819636031985283,
      "kl": 7.143774968385697,
      "learning_rate": 5e-06,
      "loss": -0.0619,
      "reward": 0.6548231065273284,
      "reward_std": 0.8737724728882312,
      "rewards/reward_func": 0.6548231065273284,
      "step": 8000,
      "toxic_reward": 4.582368350028991
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.625,
      "epoch": 1.8927221172022684,
      "format_reward": 0.0,
      "grad_norm": 7.003530025482178,
      "image_reward": 0.261529541015625,
      "kl": 9.618525552749634,
      "learning_rate": 5e-06,
      "loss": 0.051,
      "reward": -0.017962449789047243,
      "reward_std": 0.5481395080685616,
      "rewards/reward_func": -0.017962449789047243,
      "step": 8010,
      "toxic_reward": 4.220958662033081
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.3,
      "epoch": 1.8950850661625709,
      "format_reward": 0.0,
      "grad_norm": 7.718620777130127,
      "image_reward": 0.2414947509765625,
      "kl": 4.341373115777969,
      "learning_rate": 5e-06,
      "loss": 0.0835,
      "reward": 0.38290356993675234,
      "reward_std": 0.9348091699182988,
      "rewards/reward_func": 0.38290356993675234,
      "step": 8020,
      "toxic_reward": 4.002734637260437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.5,
      "epoch": 1.8974480151228734,
      "format_reward": 0.0,
      "grad_norm": 10.61853313446045,
      "image_reward": 0.2405120849609375,
      "kl": 18.692966318130495,
      "learning_rate": 5e-06,
      "loss": 0.019,
      "reward": 0.6024147510528565,
      "reward_std": 0.8250786025077105,
      "rewards/reward_func": 0.6024147510528565,
      "step": 8030,
      "toxic_reward": 4.069397926330566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.95,
      "epoch": 1.899810964083176,
      "format_reward": -0.5,
      "grad_norm": 4.87439489364624,
      "image_reward": 0.26594645231962205,
      "kl": 4.881083369255066,
      "learning_rate": 5e-06,
      "loss": 0.0141,
      "reward": 0.1986172914505005,
      "reward_std": 1.8204052031040192,
      "rewards/reward_func": 0.1986172914505005,
      "step": 8040,
      "toxic_reward": 4.146627187728882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.075,
      "epoch": 1.9021739130434783,
      "format_reward": 0.0,
      "grad_norm": 19.0607852935791,
      "image_reward": 0.2584747314453125,
      "kl": 13.449040079116822,
      "learning_rate": 5e-06,
      "loss": -0.0336,
      "reward": 0.09852480292320251,
      "reward_std": 0.37513242168352007,
      "rewards/reward_func": 0.09852480292320251,
      "step": 8050,
      "toxic_reward": 4.5937717914581295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.125,
      "epoch": 1.9045368620037806,
      "format_reward": 0.0,
      "grad_norm": 4.807636260986328,
      "image_reward": 0.238970947265625,
      "kl": 9.84277012348175,
      "learning_rate": 5e-06,
      "loss": 0.0099,
      "reward": 0.7841103792190551,
      "reward_std": 0.931809046678245,
      "rewards/reward_func": 0.7841103792190551,
      "step": 8060,
      "toxic_reward": 4.410308980941773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.575,
      "epoch": 1.9068998109640831,
      "format_reward": 0.0,
      "grad_norm": 30.570436477661133,
      "image_reward": 0.2677642822265625,
      "kl": 11.538963747024535,
      "learning_rate": 5e-06,
      "loss": 0.0571,
      "reward": 0.7513397336006165,
      "reward_std": 0.6926180317997932,
      "rewards/reward_func": 0.7513397336006165,
      "step": 8070,
      "toxic_reward": 4.325729882717132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.925,
      "epoch": 1.9092627599243857,
      "format_reward": 0.0,
      "grad_norm": 3.387159824371338,
      "image_reward": 0.2416534423828125,
      "kl": 36.42685050964356,
      "learning_rate": 5e-06,
      "loss": -0.0984,
      "reward": 1.0627863883972168,
      "reward_std": 0.9809991672635079,
      "rewards/reward_func": 1.0627863883972168,
      "step": 8080,
      "toxic_reward": 4.355536723136902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.875,
      "epoch": 1.9116257088846882,
      "format_reward": 0.0,
      "grad_norm": 8.823395729064941,
      "image_reward": 0.2377349853515625,
      "kl": 14.548263192176819,
      "learning_rate": 5e-06,
      "loss": -0.0671,
      "reward": 0.2453417807817459,
      "reward_std": 0.8620891466736793,
      "rewards/reward_func": 0.2453417807817459,
      "step": 8090,
      "toxic_reward": 4.175139570236206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.7,
      "epoch": 1.9139886578449905,
      "format_reward": -0.25,
      "grad_norm": 3.934446334838867,
      "image_reward": 0.2485321044921875,
      "kl": 476.8398398399353,
      "learning_rate": 5e-06,
      "loss": 0.1697,
      "reward": -0.03726454377174378,
      "reward_std": 1.2227270498871803,
      "rewards/reward_func": -0.03726454377174378,
      "step": 8100,
      "toxic_reward": 4.573867344856263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.8,
      "epoch": 1.916351606805293,
      "format_reward": 0.0,
      "grad_norm": 11.159741401672363,
      "image_reward": 0.2699676513671875,
      "kl": 5.77522222995758,
      "learning_rate": 5e-06,
      "loss": -0.1138,
      "reward": 0.4612575590610504,
      "reward_std": 0.5476422467269003,
      "rewards/reward_func": 0.4612575590610504,
      "step": 8110,
      "toxic_reward": 4.567614626884461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.575,
      "epoch": 1.9187145557655954,
      "format_reward": 0.0,
      "grad_norm": 11.536759376525879,
      "image_reward": 0.24012298583984376,
      "kl": 6.318757677078247,
      "learning_rate": 5e-06,
      "loss": 0.0103,
      "reward": 0.9109591245651245,
      "reward_std": 1.29407604560256,
      "rewards/reward_func": 0.9109591245651245,
      "step": 8120,
      "toxic_reward": 4.05263090133667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.575,
      "epoch": 1.9210775047258979,
      "format_reward": 0.0,
      "grad_norm": 6.833136558532715,
      "image_reward": 0.2688323974609375,
      "kl": 6.9808355331420895,
      "learning_rate": 5e-06,
      "loss": 0.0913,
      "reward": 0.9232870817184449,
      "reward_std": 0.8357461627572775,
      "rewards/reward_func": 0.9232870817184449,
      "step": 8130,
      "toxic_reward": 4.430827951431274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.9,
      "epoch": 1.9234404536862004,
      "format_reward": 0.0,
      "grad_norm": 14.8239164352417,
      "image_reward": 0.2474365234375,
      "kl": 137.28185538053512,
      "learning_rate": 5e-06,
      "loss": 0.0189,
      "reward": 0.401202654838562,
      "reward_std": 0.4000473257154226,
      "rewards/reward_func": 0.401202654838562,
      "step": 8140,
      "toxic_reward": 4.723700523376465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.175,
      "epoch": 1.925803402646503,
      "format_reward": 0.0,
      "grad_norm": 1.823515772819519,
      "image_reward": 0.22316131591796876,
      "kl": 14.130688643455505,
      "learning_rate": 5e-06,
      "loss": 0.0239,
      "reward": 1.057025855779648,
      "reward_std": 0.9014536026865244,
      "rewards/reward_func": 1.057025855779648,
      "step": 8150,
      "toxic_reward": 4.387946319580078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.875,
      "epoch": 1.9281663516068053,
      "format_reward": -0.25,
      "grad_norm": 46.756038665771484,
      "image_reward": 0.26631062775850295,
      "kl": 6.435283923149109,
      "learning_rate": 5e-06,
      "loss": -0.0328,
      "reward": 0.22599496245384215,
      "reward_std": 1.4984263110905887,
      "rewards/reward_func": 0.22599496245384215,
      "step": 8160,
      "toxic_reward": 4.138309001922607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.45,
      "epoch": 1.9305293005671076,
      "format_reward": 0.0,
      "grad_norm": 34.66867446899414,
      "image_reward": 0.25106658935546877,
      "kl": 1020.1139773368835,
      "learning_rate": 5e-06,
      "loss": 0.0999,
      "reward": 0.7446302771568298,
      "reward_std": 0.906285472586751,
      "rewards/reward_func": 0.7446302771568298,
      "step": 8170,
      "toxic_reward": 4.375624704360962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.475,
      "epoch": 1.9328922495274101,
      "format_reward": 0.0,
      "grad_norm": 2.081218957901001,
      "image_reward": 0.242840576171875,
      "kl": 3.102721667289734,
      "learning_rate": 5e-06,
      "loss": 0.0571,
      "reward": 0.5706271648406982,
      "reward_std": 0.9108416954986751,
      "rewards/reward_func": 0.5706271648406982,
      "step": 8180,
      "toxic_reward": 3.2474088430404664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.475,
      "epoch": 1.9352551984877127,
      "format_reward": -0.25,
      "grad_norm": 13.313660621643066,
      "image_reward": 0.27943929135799406,
      "kl": 9.811500716209412,
      "learning_rate": 5e-06,
      "loss": 0.0091,
      "reward": -0.0842776358127594,
      "reward_std": 1.1166115825995804,
      "rewards/reward_func": -0.0842776358127594,
      "step": 8190,
      "toxic_reward": 4.493260765075684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.225,
      "epoch": 1.9376181474480152,
      "format_reward": 0.0,
      "grad_norm": 11.93384838104248,
      "image_reward": 0.2549346923828125,
      "kl": 13.695673048496246,
      "learning_rate": 5e-06,
      "loss": -0.0083,
      "reward": 0.5832914412021637,
      "reward_std": 0.7408401468303054,
      "rewards/reward_func": 0.5832914412021637,
      "step": 8200,
      "toxic_reward": 4.143073153495789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.125,
      "epoch": 1.9399810964083177,
      "format_reward": 0.0,
      "grad_norm": 6.53907585144043,
      "image_reward": 0.2403350830078125,
      "kl": 6.522427618503571,
      "learning_rate": 5e-06,
      "loss": 0.1325,
      "reward": 0.1342033863067627,
      "reward_std": 0.7933921405114234,
      "rewards/reward_func": 0.1342033863067627,
      "step": 8210,
      "toxic_reward": 4.601714444160462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.95,
      "epoch": 1.94234404536862,
      "format_reward": 0.0,
      "grad_norm": 23.774093627929688,
      "image_reward": 0.25664520263671875,
      "kl": 5.8061746001243595,
      "learning_rate": 5e-06,
      "loss": -0.1029,
      "reward": 0.6099749207496643,
      "reward_std": 1.0578389540314674,
      "rewards/reward_func": 0.6099749207496643,
      "step": 8220,
      "toxic_reward": 3.542074370384216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.525,
      "epoch": 1.9447069943289224,
      "format_reward": 0.0,
      "grad_norm": 19.021333694458008,
      "image_reward": 0.25049285888671874,
      "kl": 4.400176310539246,
      "learning_rate": 5e-06,
      "loss": -0.0805,
      "reward": 0.271647572517395,
      "reward_std": 0.8572761943563819,
      "rewards/reward_func": 0.271647572517395,
      "step": 8230,
      "toxic_reward": 4.576322746276856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.4,
      "epoch": 1.947069943289225,
      "format_reward": -0.25,
      "grad_norm": 12.740744590759277,
      "image_reward": 0.2639495849609375,
      "kl": 53.72892454862595,
      "learning_rate": 5e-06,
      "loss": 0.0517,
      "reward": 0.4684752345085144,
      "reward_std": 1.5598361855372787,
      "rewards/reward_func": 0.4684752345085144,
      "step": 8240,
      "toxic_reward": 4.280627131462097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.05,
      "epoch": 1.9494328922495274,
      "format_reward": 0.0,
      "grad_norm": 8.727499961853027,
      "image_reward": 0.273443603515625,
      "kl": 9.401781392097472,
      "learning_rate": 5e-06,
      "loss": -0.0578,
      "reward": 0.6043965280056,
      "reward_std": 0.7762668525800109,
      "rewards/reward_func": 0.6043965280056,
      "step": 8250,
      "toxic_reward": 4.007175719738006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.175,
      "epoch": 1.95179584120983,
      "format_reward": -0.25,
      "grad_norm": 21.95665740966797,
      "image_reward": 0.2783833831548691,
      "kl": 6.502747631072998,
      "learning_rate": 5e-06,
      "loss": -0.1757,
      "reward": 0.7646288216114044,
      "reward_std": 1.2125793328508734,
      "rewards/reward_func": 0.7646288216114044,
      "step": 8260,
      "toxic_reward": 4.438870096206665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.375,
      "epoch": 1.9541587901701323,
      "format_reward": 0.0,
      "grad_norm": 19.78591537475586,
      "image_reward": 0.2660888671875,
      "kl": 101.96959731578826,
      "learning_rate": 5e-06,
      "loss": 0.0584,
      "reward": 0.8457072794437408,
      "reward_std": 0.8602423138916493,
      "rewards/reward_func": 0.8457072794437408,
      "step": 8270,
      "toxic_reward": 4.274328458309173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.325,
      "epoch": 1.9565217391304348,
      "format_reward": -0.25,
      "grad_norm": 7.575157642364502,
      "image_reward": 0.26166178435087206,
      "kl": 7.8605184674263,
      "learning_rate": 5e-06,
      "loss": -0.0592,
      "reward": 0.7780414521694183,
      "reward_std": 1.34521058909595,
      "rewards/reward_func": 0.7780414521694183,
      "step": 8280,
      "toxic_reward": 4.395621502399445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.0,
      "epoch": 1.9588846880907371,
      "format_reward": 0.0,
      "grad_norm": 13.91838550567627,
      "image_reward": 0.2494293212890625,
      "kl": 3.4681380152702332,
      "learning_rate": 5e-06,
      "loss": -0.0285,
      "reward": 1.0126874148845673,
      "reward_std": 0.884580178745091,
      "rewards/reward_func": 1.0126874148845673,
      "step": 8290,
      "toxic_reward": 4.259213161468506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.275,
      "epoch": 1.9612476370510397,
      "format_reward": -0.5,
      "grad_norm": 11.346104621887207,
      "image_reward": 0.24504598081111909,
      "kl": 17.73236060142517,
      "learning_rate": 5e-06,
      "loss": -0.0501,
      "reward": -0.41139370799064634,
      "reward_std": 1.535068777576089,
      "rewards/reward_func": -0.41139370799064634,
      "step": 8300,
      "toxic_reward": 4.184125363826752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.1,
      "epoch": 1.9636105860113422,
      "format_reward": 0.0,
      "grad_norm": 7.8980631828308105,
      "image_reward": 0.2494049072265625,
      "kl": 1.3982277452945708,
      "learning_rate": 5e-06,
      "loss": -0.0632,
      "reward": 0.7493218898773193,
      "reward_std": 0.7001253291964531,
      "rewards/reward_func": 0.7493218898773193,
      "step": 8310,
      "toxic_reward": 4.593434143066406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.025,
      "epoch": 1.9659735349716447,
      "format_reward": 0.0,
      "grad_norm": 1.629384994506836,
      "image_reward": 0.2574554443359375,
      "kl": 9.406988048553467,
      "learning_rate": 5e-06,
      "loss": 0.0561,
      "reward": 0.6752925157546997,
      "reward_std": 1.2529858350753784,
      "rewards/reward_func": 0.6752925157546997,
      "step": 8320,
      "toxic_reward": 3.6643527030944822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.725,
      "epoch": 1.968336483931947,
      "format_reward": 0.0,
      "grad_norm": 6.693783283233643,
      "image_reward": 0.24935455322265626,
      "kl": 4.6708708822727205,
      "learning_rate": 5e-06,
      "loss": -0.0369,
      "reward": 1.2317909479141236,
      "reward_std": 1.4201693460345268,
      "rewards/reward_func": 1.2317909479141236,
      "step": 8330,
      "toxic_reward": 3.705500102043152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.675,
      "epoch": 1.9706994328922496,
      "format_reward": 0.0,
      "grad_norm": 13.678855895996094,
      "image_reward": 0.26453857421875,
      "kl": 1.7596666514873505,
      "learning_rate": 5e-06,
      "loss": 0.0084,
      "reward": 0.6438661813735962,
      "reward_std": 0.5453263748437166,
      "rewards/reward_func": 0.6438661813735962,
      "step": 8340,
      "toxic_reward": 4.577846193313599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.55,
      "epoch": 1.973062381852552,
      "format_reward": 0.0,
      "grad_norm": 5.530174255371094,
      "image_reward": 0.2434234619140625,
      "kl": 16.00339319705963,
      "learning_rate": 5e-06,
      "loss": 0.0858,
      "reward": 0.7399854481220245,
      "reward_std": 0.5954274158924818,
      "rewards/reward_func": 0.7399854481220245,
      "step": 8350,
      "toxic_reward": 4.567293620109558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.8,
      "epoch": 1.9754253308128544,
      "format_reward": -0.25,
      "grad_norm": 23.65260124206543,
      "image_reward": 0.2397003173828125,
      "kl": 413.27391294240954,
      "learning_rate": 5e-06,
      "loss": -0.0452,
      "reward": 0.21110110878944396,
      "reward_std": 1.2717279449105263,
      "rewards/reward_func": 0.21110110878944396,
      "step": 8360,
      "toxic_reward": 4.29474036693573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.15,
      "epoch": 1.977788279773157,
      "format_reward": 0.0,
      "grad_norm": 8.489328384399414,
      "image_reward": 0.2703460693359375,
      "kl": 11.292006134986877,
      "learning_rate": 5e-06,
      "loss": -0.0396,
      "reward": 0.522923594713211,
      "reward_std": 0.6722989223897458,
      "rewards/reward_func": 0.522923594713211,
      "step": 8370,
      "toxic_reward": 4.362267994880677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.7,
      "epoch": 1.9801512287334595,
      "format_reward": 0.0,
      "grad_norm": 14.112800598144531,
      "image_reward": 0.2404998779296875,
      "kl": 5.943003642559051,
      "learning_rate": 5e-06,
      "loss": 0.0066,
      "reward": 1.046756339073181,
      "reward_std": 1.401267148554325,
      "rewards/reward_func": 1.046756339073181,
      "step": 8380,
      "toxic_reward": 4.379712152481079
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.075,
      "epoch": 1.9825141776937618,
      "format_reward": 0.0,
      "grad_norm": 15.351452827453613,
      "image_reward": 0.2679229736328125,
      "kl": 2.1231451511383055,
      "learning_rate": 5e-06,
      "loss": -0.0874,
      "reward": 0.044296592473983765,
      "reward_std": 0.7907688375562429,
      "rewards/reward_func": 0.044296592473983765,
      "step": 8390,
      "toxic_reward": 4.44194188117981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.8,
      "epoch": 1.9848771266540641,
      "format_reward": 0.0,
      "grad_norm": 14.493269920349121,
      "image_reward": 0.23359222412109376,
      "kl": 15.598973235487938,
      "learning_rate": 5e-06,
      "loss": 0.0523,
      "reward": 0.6035852313041687,
      "reward_std": 0.7898097388446331,
      "rewards/reward_func": 0.6035852313041687,
      "step": 8400,
      "toxic_reward": 4.0595218420028685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.025,
      "epoch": 1.9872400756143667,
      "format_reward": 0.0,
      "grad_norm": 2.004755735397339,
      "image_reward": 0.23458099365234375,
      "kl": 13.407473123073578,
      "learning_rate": 5e-06,
      "loss": 0.0054,
      "reward": 0.5494411200284958,
      "reward_std": 0.5586541540920734,
      "rewards/reward_func": 0.5494411200284958,
      "step": 8410,
      "toxic_reward": 4.175926774740219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.325,
      "epoch": 1.9896030245746692,
      "format_reward": 0.0,
      "grad_norm": 9.598527908325195,
      "image_reward": 0.301202392578125,
      "kl": 4.9204403221607205,
      "learning_rate": 5e-06,
      "loss": -0.001,
      "reward": 0.4649462789297104,
      "reward_std": 0.7171205889433623,
      "rewards/reward_func": 0.4649462789297104,
      "step": 8420,
      "toxic_reward": 3.910860872268677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 57.825,
      "epoch": 1.9919659735349717,
      "format_reward": 0.0,
      "grad_norm": 2.607607841491699,
      "image_reward": 0.2745025634765625,
      "kl": 9.545298218727112,
      "learning_rate": 5e-06,
      "loss": 0.0779,
      "reward": 0.43806184232234957,
      "reward_std": 0.8561135273426771,
      "rewards/reward_func": 0.43806184232234957,
      "step": 8430,
      "toxic_reward": 4.119659066200256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.8,
      "epoch": 1.994328922495274,
      "format_reward": -0.25,
      "grad_norm": 1.9090756177902222,
      "image_reward": 0.264605712890625,
      "kl": 1.142916288971901,
      "learning_rate": 5e-06,
      "loss": -0.0035,
      "reward": -0.06725225448608399,
      "reward_std": 1.1679431475698947,
      "rewards/reward_func": -0.06725225448608399,
      "step": 8440,
      "toxic_reward": 4.506508493423462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.625,
      "epoch": 1.9966918714555766,
      "format_reward": -0.25,
      "grad_norm": 1.924688458442688,
      "image_reward": 0.247418212890625,
      "kl": 1.739441803097725,
      "learning_rate": 5e-06,
      "loss": -0.0847,
      "reward": 0.2795759916305542,
      "reward_std": 1.532812624052167,
      "rewards/reward_func": 0.2795759916305542,
      "step": 8450,
      "toxic_reward": 3.7154327273368835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.075,
      "epoch": 1.999054820415879,
      "format_reward": 0.0,
      "grad_norm": 3.183807373046875,
      "image_reward": 0.259228515625,
      "kl": 1.071340024471283,
      "learning_rate": 5e-06,
      "loss": 0.0299,
      "reward": 1.3993828475475312,
      "reward_std": 1.1979968290776015,
      "rewards/reward_func": 1.3993828475475312,
      "step": 8460,
      "toxic_reward": 4.236328482627869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.325,
      "epoch": 2.0014177693761814,
      "format_reward": 0.0,
      "grad_norm": 7.500320911407471,
      "image_reward": 0.2599090576171875,
      "kl": 1.2782041728496552,
      "learning_rate": 5e-06,
      "loss": 0.046,
      "reward": 1.2368434906005858,
      "reward_std": 1.188733378984034,
      "rewards/reward_func": 1.2368434906005858,
      "step": 8470,
      "toxic_reward": 3.8694416284561157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.575,
      "epoch": 2.003780718336484,
      "format_reward": 0.0,
      "grad_norm": 3.4954817295074463,
      "image_reward": 0.25406494140625,
      "kl": 2.6761809453368186,
      "learning_rate": 5e-06,
      "loss": -0.0561,
      "reward": 0.3607616722583771,
      "reward_std": 0.599818766117096,
      "rewards/reward_func": 0.3607616722583771,
      "step": 8480,
      "toxic_reward": 4.048572421073914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.05,
      "epoch": 2.0061436672967865,
      "format_reward": 0.0,
      "grad_norm": 5.18286657333374,
      "image_reward": 0.22822036743164062,
      "kl": 2.461097413301468,
      "learning_rate": 5e-06,
      "loss": -0.0261,
      "reward": 0.2195432722568512,
      "reward_std": 0.7936036609113216,
      "rewards/reward_func": 0.2195432722568512,
      "step": 8490,
      "toxic_reward": 4.110178589820862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.9,
      "epoch": 2.008506616257089,
      "format_reward": -0.75,
      "grad_norm": 2.6953821182250977,
      "image_reward": 0.238427734375,
      "kl": 1.0251432090997696,
      "learning_rate": 5e-06,
      "loss": -0.0027,
      "reward": -0.4569409370422363,
      "reward_std": 1.0821652268990873,
      "rewards/reward_func": -0.4569409370422363,
      "step": 8500,
      "toxic_reward": 4.185848736763001
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.4,
      "epoch": 2.010869565217391,
      "format_reward": 0.0,
      "grad_norm": 6.174482822418213,
      "image_reward": 0.245025634765625,
      "kl": 570.9768789380789,
      "learning_rate": 5e-06,
      "loss": 0.0155,
      "reward": 0.634968101978302,
      "reward_std": 0.5698891028761863,
      "rewards/reward_func": 0.634968101978302,
      "step": 8510,
      "toxic_reward": 4.557809638977051
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.525,
      "epoch": 2.0132325141776937,
      "format_reward": 0.0,
      "grad_norm": 12.716261863708496,
      "image_reward": 0.2716217041015625,
      "kl": 1.0744814962148665,
      "learning_rate": 5e-06,
      "loss": 0.0371,
      "reward": 0.8971363306045532,
      "reward_std": 1.0540940549224616,
      "rewards/reward_func": 0.8971363306045532,
      "step": 8520,
      "toxic_reward": 4.03425624370575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.725,
      "epoch": 2.015595463137996,
      "format_reward": 0.0,
      "grad_norm": 11.573805809020996,
      "image_reward": 0.25701904296875,
      "kl": 1.1612621247768402,
      "learning_rate": 5e-06,
      "loss": -0.0256,
      "reward": 0.4974235534667969,
      "reward_std": 0.7099893309175969,
      "rewards/reward_func": 0.4974235534667969,
      "step": 8530,
      "toxic_reward": 4.757012367248535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.325,
      "epoch": 2.0179584120982987,
      "format_reward": 0.0,
      "grad_norm": 2.5829172134399414,
      "image_reward": 0.23183441162109375,
      "kl": 1.0122918039560318,
      "learning_rate": 5e-06,
      "loss": -0.0224,
      "reward": 0.5489160656929016,
      "reward_std": 0.4481811560690403,
      "rewards/reward_func": 0.5489160656929016,
      "step": 8540,
      "toxic_reward": 4.330148541927338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.6,
      "epoch": 2.0203213610586013,
      "format_reward": -0.25,
      "grad_norm": 8.287252426147461,
      "image_reward": 0.2674835205078125,
      "kl": 1.138858178257942,
      "learning_rate": 5e-06,
      "loss": 0.0238,
      "reward": -0.09747375845909119,
      "reward_std": 0.8301142632961274,
      "rewards/reward_func": -0.09747375845909119,
      "step": 8550,
      "toxic_reward": 4.7045900344848635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.65,
      "epoch": 2.022684310018904,
      "format_reward": 0.0,
      "grad_norm": 9.928176879882812,
      "image_reward": 0.2459930419921875,
      "kl": 1.701068675518036,
      "learning_rate": 5e-06,
      "loss": -0.0446,
      "reward": 0.5473175823688508,
      "reward_std": 0.7223521884530782,
      "rewards/reward_func": 0.5473175823688508,
      "step": 8560,
      "toxic_reward": 4.571657824516296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.725,
      "epoch": 2.025047258979206,
      "format_reward": -0.25,
      "grad_norm": 5.9600677490234375,
      "image_reward": 0.26257222443819045,
      "kl": 2.5904053121805193,
      "learning_rate": 5e-06,
      "loss": 0.0163,
      "reward": -0.20251348614692688,
      "reward_std": 0.9303808398544788,
      "rewards/reward_func": -0.20251348614692688,
      "step": 8570,
      "toxic_reward": 4.581225419044495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.55,
      "epoch": 2.0274102079395084,
      "format_reward": 0.0,
      "grad_norm": 3.309791088104248,
      "image_reward": 0.23095855712890626,
      "kl": 1.5135916233062745,
      "learning_rate": 5e-06,
      "loss": -0.0068,
      "reward": 0.21151033639907837,
      "reward_std": 0.7603108703624457,
      "rewards/reward_func": 0.21151033639907837,
      "step": 8580,
      "toxic_reward": 4.328943312168121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.5,
      "epoch": 2.029773156899811,
      "format_reward": 0.0,
      "grad_norm": 8.408251762390137,
      "image_reward": 0.2651763916015625,
      "kl": 0.6560351371765136,
      "learning_rate": 5e-06,
      "loss": -0.0997,
      "reward": 0.012960964441299438,
      "reward_std": 0.35295800119638443,
      "rewards/reward_func": 0.012960964441299438,
      "step": 8590,
      "toxic_reward": 4.5852957487106325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.425,
      "epoch": 2.0321361058601135,
      "format_reward": 0.0,
      "grad_norm": 10.234503746032715,
      "image_reward": 0.242572021484375,
      "kl": 1.6390519708395004,
      "learning_rate": 5e-06,
      "loss": -0.0652,
      "reward": 0.49767774939537046,
      "reward_std": 0.8766103692352771,
      "rewards/reward_func": 0.49767774939537046,
      "step": 8600,
      "toxic_reward": 4.273276591300965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.525,
      "epoch": 2.034499054820416,
      "format_reward": 0.0,
      "grad_norm": 31.749767303466797,
      "image_reward": 0.24173736572265625,
      "kl": 1.4096274197101593,
      "learning_rate": 5e-06,
      "loss": -0.0081,
      "reward": 0.4087996184825897,
      "reward_std": 0.9022964790463448,
      "rewards/reward_func": 0.4087996184825897,
      "step": 8610,
      "toxic_reward": 4.324217915534973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.825,
      "epoch": 2.036862003780718,
      "format_reward": -0.25,
      "grad_norm": 12.874961853027344,
      "image_reward": 0.2652323395013809,
      "kl": 1.0484755635261536,
      "learning_rate": 5e-06,
      "loss": 0.027,
      "reward": -0.2117618590593338,
      "reward_std": 1.249949687719345,
      "rewards/reward_func": -0.2117618590593338,
      "step": 8620,
      "toxic_reward": 3.5184057116508485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.375,
      "epoch": 2.0392249527410207,
      "format_reward": 0.0,
      "grad_norm": 1.5234144926071167,
      "image_reward": 0.2357269287109375,
      "kl": 1.2848842471837998,
      "learning_rate": 5e-06,
      "loss": 0.0752,
      "reward": 0.716532975435257,
      "reward_std": 0.89201683960855,
      "rewards/reward_func": 0.716532975435257,
      "step": 8630,
      "toxic_reward": 4.4441750049591064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.125,
      "epoch": 2.041587901701323,
      "format_reward": 0.0,
      "grad_norm": 1.1807531118392944,
      "image_reward": 0.25293731689453125,
      "kl": 2.8808428183197976,
      "learning_rate": 5e-06,
      "loss": 0.023,
      "reward": 0.9091297924518585,
      "reward_std": 0.7464996237307787,
      "rewards/reward_func": 0.9091297924518585,
      "step": 8640,
      "toxic_reward": 4.224450874328613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.15,
      "epoch": 2.0439508506616257,
      "format_reward": 0.0,
      "grad_norm": 10.838650703430176,
      "image_reward": 0.23937225341796875,
      "kl": 2.5997998148202894,
      "learning_rate": 5e-06,
      "loss": -0.0419,
      "reward": 0.3074700653553009,
      "reward_std": 0.8474891871213913,
      "rewards/reward_func": 0.3074700653553009,
      "step": 8650,
      "toxic_reward": 4.419002604484558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.3,
      "epoch": 2.0463137996219283,
      "format_reward": 0.0,
      "grad_norm": 2.063800811767578,
      "image_reward": 0.2351318359375,
      "kl": 2.429117688536644,
      "learning_rate": 5e-06,
      "loss": 0.0507,
      "reward": 0.464035177230835,
      "reward_std": 0.8178490117192269,
      "rewards/reward_func": 0.464035177230835,
      "step": 8660,
      "toxic_reward": 4.122921991348266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.725,
      "epoch": 2.048676748582231,
      "format_reward": -0.25,
      "grad_norm": 5.179421424865723,
      "image_reward": 0.2340398147702217,
      "kl": 1.4141836494207383,
      "learning_rate": 5e-06,
      "loss": -0.0321,
      "reward": 0.2651833713054657,
      "reward_std": 1.3055690463632346,
      "rewards/reward_func": 0.2651833713054657,
      "step": 8670,
      "toxic_reward": 4.099702596664429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.0,
      "epoch": 2.051039697542533,
      "format_reward": 0.0,
      "grad_norm": 5.729818344116211,
      "image_reward": 0.22962646484375,
      "kl": 1.0476927325129508,
      "learning_rate": 5e-06,
      "loss": 0.0396,
      "reward": 0.5858624681830407,
      "reward_std": 0.8647454358637333,
      "rewards/reward_func": 0.5858624681830407,
      "step": 8680,
      "toxic_reward": 3.923676002025604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.575,
      "epoch": 2.0534026465028354,
      "format_reward": -0.25,
      "grad_norm": 33.29255294799805,
      "image_reward": 0.23504893034696578,
      "kl": 0.5507122159004212,
      "learning_rate": 5e-06,
      "loss": 0.0042,
      "reward": 0.5358553946018219,
      "reward_std": 1.371009534597397,
      "rewards/reward_func": 0.5358553946018219,
      "step": 8690,
      "toxic_reward": 4.1810872793197635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.425,
      "epoch": 2.055765595463138,
      "format_reward": 0.0,
      "grad_norm": 9.838844299316406,
      "image_reward": 0.2368377685546875,
      "kl": 0.8958913296461105,
      "learning_rate": 5e-06,
      "loss": -0.0191,
      "reward": 0.8707101225852967,
      "reward_std": 1.2157209530472755,
      "rewards/reward_func": 0.8707101225852967,
      "step": 8700,
      "toxic_reward": 3.7892824172973634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.275,
      "epoch": 2.0581285444234405,
      "format_reward": 0.0,
      "grad_norm": 18.979665756225586,
      "image_reward": 0.259954833984375,
      "kl": 1.8286799043416977,
      "learning_rate": 5e-06,
      "loss": -0.0646,
      "reward": 0.3829235196113586,
      "reward_std": 0.9690108880400657,
      "rewards/reward_func": 0.3829235196113586,
      "step": 8710,
      "toxic_reward": 4.264838469028473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.35,
      "epoch": 2.060491493383743,
      "format_reward": 0.0,
      "grad_norm": 6.838248252868652,
      "image_reward": 0.22672042846679688,
      "kl": 0.7338828861713409,
      "learning_rate": 5e-06,
      "loss": 0.0185,
      "reward": 0.827715927362442,
      "reward_std": 0.9109129812568426,
      "rewards/reward_func": 0.827715927362442,
      "step": 8720,
      "toxic_reward": 4.2410869836807255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.725,
      "epoch": 2.0628544423440456,
      "format_reward": 0.0,
      "grad_norm": 4.239810943603516,
      "image_reward": 0.25128173828125,
      "kl": 5.776644492149353,
      "learning_rate": 5e-06,
      "loss": -0.0179,
      "reward": 0.5484015077352524,
      "reward_std": 1.311685237288475,
      "rewards/reward_func": 0.5484015077352524,
      "step": 8730,
      "toxic_reward": 4.056830906867981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.75,
      "epoch": 2.0652173913043477,
      "format_reward": 0.0,
      "grad_norm": 11.002195358276367,
      "image_reward": 0.229833984375,
      "kl": 1.1322214603424072,
      "learning_rate": 5e-06,
      "loss": 0.1107,
      "reward": -0.03462121486663818,
      "reward_std": 0.43875638470053674,
      "rewards/reward_func": -0.03462121486663818,
      "step": 8740,
      "toxic_reward": 4.620968174934387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.975,
      "epoch": 2.06758034026465,
      "format_reward": -0.25,
      "grad_norm": 11.761068344116211,
      "image_reward": 0.227862548828125,
      "kl": 7.682415267825126,
      "learning_rate": 5e-06,
      "loss": -0.09,
      "reward": 0.26341341733932494,
      "reward_std": 1.4870022028684615,
      "rewards/reward_func": 0.26341341733932494,
      "step": 8750,
      "toxic_reward": 4.433785676956177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.425,
      "epoch": 2.0699432892249527,
      "format_reward": 0.0,
      "grad_norm": 5.119037628173828,
      "image_reward": 0.2395660400390625,
      "kl": 1.2481903672218322,
      "learning_rate": 5e-06,
      "loss": -0.0746,
      "reward": 0.3276951313018799,
      "reward_std": 0.46017137840390204,
      "rewards/reward_func": 0.3276951313018799,
      "step": 8760,
      "toxic_reward": 4.593200016021728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.775,
      "epoch": 2.0723062381852553,
      "format_reward": 0.0,
      "grad_norm": 1.704590916633606,
      "image_reward": 0.2639801025390625,
      "kl": 2.778309851884842,
      "learning_rate": 5e-06,
      "loss": -0.0417,
      "reward": 0.30720534920692444,
      "reward_std": 0.6144355796277523,
      "rewards/reward_func": 0.30720534920692444,
      "step": 8770,
      "toxic_reward": 4.481031203269959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.75,
      "epoch": 2.074669187145558,
      "format_reward": 0.0,
      "grad_norm": 11.392171859741211,
      "image_reward": 0.2711700439453125,
      "kl": 2.4740293115377425,
      "learning_rate": 5e-06,
      "loss": 0.0428,
      "reward": 0.318191659450531,
      "reward_std": 0.6928373419679701,
      "rewards/reward_func": 0.318191659450531,
      "step": 8780,
      "toxic_reward": 4.502946138381958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.85,
      "epoch": 2.07703213610586,
      "format_reward": 0.0,
      "grad_norm": 11.103386878967285,
      "image_reward": 0.2586761474609375,
      "kl": 0.8142087966203689,
      "learning_rate": 5e-06,
      "loss": -0.0231,
      "reward": 0.5202795565128326,
      "reward_std": 0.9188865400850773,
      "rewards/reward_func": 0.5202795565128326,
      "step": 8790,
      "toxic_reward": 4.268853735923767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.825,
      "epoch": 2.0793950850661624,
      "format_reward": -0.5,
      "grad_norm": 3.831815719604492,
      "image_reward": 0.23363494873046875,
      "kl": 3.2880129516124725,
      "learning_rate": 5e-06,
      "loss": 0.0507,
      "reward": -0.17698687314987183,
      "reward_std": 1.4326126247644424,
      "rewards/reward_func": -0.17698687314987183,
      "step": 8800,
      "toxic_reward": 4.581766486167908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.95,
      "epoch": 2.081758034026465,
      "format_reward": -0.5,
      "grad_norm": 13.328118324279785,
      "image_reward": 0.283209228515625,
      "kl": 2.583532452583313,
      "learning_rate": 5e-06,
      "loss": 0.1117,
      "reward": 0.2989026606082916,
      "reward_std": 2.0480130195617674,
      "rewards/reward_func": 0.2989026606082916,
      "step": 8810,
      "toxic_reward": 3.5884770512580872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.125,
      "epoch": 2.0841209829867675,
      "format_reward": 0.0,
      "grad_norm": 1.780765414237976,
      "image_reward": 0.242584228515625,
      "kl": 1.3252637952566146,
      "learning_rate": 5e-06,
      "loss": -0.0244,
      "reward": 0.29446207284927367,
      "reward_std": 1.0302367629483342,
      "rewards/reward_func": 0.29446207284927367,
      "step": 8820,
      "toxic_reward": 4.3270234823226925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.55,
      "epoch": 2.08648393194707,
      "format_reward": 0.0,
      "grad_norm": 0.9498596787452698,
      "image_reward": 0.25778045654296877,
      "kl": 1.5434826999902724,
      "learning_rate": 5e-06,
      "loss": -0.1175,
      "reward": 0.414847657084465,
      "reward_std": 0.7169051881879568,
      "rewards/reward_func": 0.414847657084465,
      "step": 8830,
      "toxic_reward": 4.253011137247086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.925,
      "epoch": 2.0888468809073726,
      "format_reward": 0.0,
      "grad_norm": 8.519845008850098,
      "image_reward": 0.23509521484375,
      "kl": 0.5451234139502048,
      "learning_rate": 5e-06,
      "loss": 0.1295,
      "reward": 0.510816776752472,
      "reward_std": 0.6249840931501239,
      "rewards/reward_func": 0.510816776752472,
      "step": 8840,
      "toxic_reward": 4.746308994293213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.225,
      "epoch": 2.0912098298676747,
      "format_reward": 0.0,
      "grad_norm": 3.2223246097564697,
      "image_reward": 0.23397216796875,
      "kl": 1.196854567527771,
      "learning_rate": 5e-06,
      "loss": 0.0121,
      "reward": 0.430766886472702,
      "reward_std": 1.1830935038626194,
      "rewards/reward_func": 0.430766886472702,
      "step": 8850,
      "toxic_reward": 4.010055112838745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.1,
      "epoch": 2.093572778827977,
      "format_reward": 0.0,
      "grad_norm": 4.516735553741455,
      "image_reward": 0.248101806640625,
      "kl": 1.3512360364198686,
      "learning_rate": 5e-06,
      "loss": -0.0064,
      "reward": 0.4724120795726776,
      "reward_std": 1.2229724466800689,
      "rewards/reward_func": 0.4724120795726776,
      "step": 8860,
      "toxic_reward": 4.219368410110474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.45,
      "epoch": 2.0959357277882797,
      "format_reward": 0.0,
      "grad_norm": 10.528098106384277,
      "image_reward": 0.2487030029296875,
      "kl": 0.8361154735088349,
      "learning_rate": 5e-06,
      "loss": 0.0096,
      "reward": 0.481815043091774,
      "reward_std": 0.8121814839541912,
      "rewards/reward_func": 0.481815043091774,
      "step": 8870,
      "toxic_reward": 4.385925316810608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.625,
      "epoch": 2.0982986767485823,
      "format_reward": 0.0,
      "grad_norm": 14.012845993041992,
      "image_reward": 0.26658477783203127,
      "kl": 5.801792293787003,
      "learning_rate": 5e-06,
      "loss": -0.0543,
      "reward": 0.8176810801029205,
      "reward_std": 0.9641741991043091,
      "rewards/reward_func": 0.8176810801029205,
      "step": 8880,
      "toxic_reward": 4.4868937015533445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 58.625,
      "epoch": 2.100661625708885,
      "format_reward": 0.0,
      "grad_norm": 1.7836425304412842,
      "image_reward": 0.219366455078125,
      "kl": 2.3325648605823517,
      "learning_rate": 5e-06,
      "loss": -0.0747,
      "reward": 0.07698584794998169,
      "reward_std": 0.7939232878386975,
      "rewards/reward_func": 0.07698584794998169,
      "step": 8890,
      "toxic_reward": 3.8968687295913695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.1,
      "epoch": 2.1030245746691874,
      "format_reward": 0.0,
      "grad_norm": 4.322965145111084,
      "image_reward": 0.2381072998046875,
      "kl": 1.5042289346456528,
      "learning_rate": 5e-06,
      "loss": -0.0396,
      "reward": 0.6448795169591903,
      "reward_std": 1.071408730885014,
      "rewards/reward_func": 0.6448795169591903,
      "step": 8900,
      "toxic_reward": 4.1689093708992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.6,
      "epoch": 2.1053875236294894,
      "format_reward": -0.25,
      "grad_norm": 3.794384002685547,
      "image_reward": 0.2495122268795967,
      "kl": 1.5198310285806655,
      "learning_rate": 5e-06,
      "loss": 0.0187,
      "reward": -0.018249320983886718,
      "reward_std": 0.9345291556790472,
      "rewards/reward_func": -0.018249320983886718,
      "step": 8910,
      "toxic_reward": 4.52795147895813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.425,
      "epoch": 2.107750472589792,
      "format_reward": 0.0,
      "grad_norm": 5.023624420166016,
      "image_reward": 0.2479766845703125,
      "kl": 0.8687845975160599,
      "learning_rate": 5e-06,
      "loss": -0.0294,
      "reward": 0.188352632522583,
      "reward_std": 0.6677849385887384,
      "rewards/reward_func": 0.188352632522583,
      "step": 8920,
      "toxic_reward": 4.531054210662842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.625,
      "epoch": 2.1101134215500945,
      "format_reward": 0.0,
      "grad_norm": 2.3595223426818848,
      "image_reward": 0.2230987548828125,
      "kl": 8.144508588314057,
      "learning_rate": 5e-06,
      "loss": -0.0584,
      "reward": 0.1267090529203415,
      "reward_std": 0.4783048752695322,
      "rewards/reward_func": 0.1267090529203415,
      "step": 8930,
      "toxic_reward": 4.553759598731995
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.075,
      "epoch": 2.112476370510397,
      "format_reward": 0.0,
      "grad_norm": 27.809850692749023,
      "image_reward": 0.2523773193359375,
      "kl": 0.7018902823328972,
      "learning_rate": 5e-06,
      "loss": -0.0582,
      "reward": 0.3417531728744507,
      "reward_std": 0.6842773109674454,
      "rewards/reward_func": 0.3417531728744507,
      "step": 8940,
      "toxic_reward": 4.307323157787323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.875,
      "epoch": 2.1148393194706996,
      "format_reward": -0.25,
      "grad_norm": 2.785470962524414,
      "image_reward": 0.2328338623046875,
      "kl": 0.866449561715126,
      "learning_rate": 5e-06,
      "loss": -0.0235,
      "reward": 0.9232547760009766,
      "reward_std": 1.3794653311371803,
      "rewards/reward_func": 0.9232547760009766,
      "step": 8950,
      "toxic_reward": 4.619945740699768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.5,
      "epoch": 2.1172022684310017,
      "format_reward": 0.0,
      "grad_norm": 7.097494602203369,
      "image_reward": 0.2396942138671875,
      "kl": 2.0439702540636064,
      "learning_rate": 5e-06,
      "loss": 0.072,
      "reward": 0.26967796087265017,
      "reward_std": 0.541577224060893,
      "rewards/reward_func": 0.26967796087265017,
      "step": 8960,
      "toxic_reward": 4.525745010375976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.5,
      "epoch": 2.119565217391304,
      "format_reward": 0.0,
      "grad_norm": 9.064950942993164,
      "image_reward": 0.256915283203125,
      "kl": 1.2320655643939973,
      "learning_rate": 5e-06,
      "loss": -0.0406,
      "reward": 0.2311327040195465,
      "reward_std": 0.5596455704420805,
      "rewards/reward_func": 0.2311327040195465,
      "step": 8970,
      "toxic_reward": 4.595108699798584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.65,
      "epoch": 2.1219281663516067,
      "format_reward": 0.0,
      "grad_norm": 2.589348793029785,
      "image_reward": 0.2637451171875,
      "kl": 3.0987811207771303,
      "learning_rate": 5e-06,
      "loss": 0.0007,
      "reward": 0.11190776824951172,
      "reward_std": 0.6281056736595929,
      "rewards/reward_func": 0.11190776824951172,
      "step": 8980,
      "toxic_reward": 3.8689257740974425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.575,
      "epoch": 2.1242911153119093,
      "format_reward": 0.0,
      "grad_norm": 8.932865142822266,
      "image_reward": 0.246466064453125,
      "kl": 0.9722367227077484,
      "learning_rate": 5e-06,
      "loss": -0.0272,
      "reward": 0.507748281955719,
      "reward_std": 0.49981794953346254,
      "rewards/reward_func": 0.507748281955719,
      "step": 8990,
      "toxic_reward": 4.460081267356872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.45,
      "epoch": 2.126654064272212,
      "format_reward": 0.0,
      "grad_norm": 6.9373064041137695,
      "image_reward": 0.247686767578125,
      "kl": 1.1410144418478012,
      "learning_rate": 5e-06,
      "loss": -0.0068,
      "reward": 0.7415260970592499,
      "reward_std": 0.7849105328321457,
      "rewards/reward_func": 0.7415260970592499,
      "step": 9000,
      "toxic_reward": 4.300376343727112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.45,
      "epoch": 2.1290170132325144,
      "format_reward": 0.0,
      "grad_norm": 2.5460894107818604,
      "image_reward": 0.253936767578125,
      "kl": 0.8742299884557724,
      "learning_rate": 5e-06,
      "loss": 0.009,
      "reward": 0.2949145630002022,
      "reward_std": 0.8127535484731198,
      "rewards/reward_func": 0.2949145630002022,
      "step": 9010,
      "toxic_reward": 4.208229756355285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.875,
      "epoch": 2.1313799621928164,
      "format_reward": 0.0,
      "grad_norm": 8.273791313171387,
      "image_reward": 0.23465576171875,
      "kl": 1.841402593255043,
      "learning_rate": 5e-06,
      "loss": 0.0698,
      "reward": 0.2962026834487915,
      "reward_std": 0.5367021195590496,
      "rewards/reward_func": 0.2962026834487915,
      "step": 9020,
      "toxic_reward": 4.80813364982605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.275,
      "epoch": 2.133742911153119,
      "format_reward": 0.0,
      "grad_norm": 1.396345853805542,
      "image_reward": 0.25015411376953123,
      "kl": 2.9800774693489074,
      "learning_rate": 5e-06,
      "loss": 0.0132,
      "reward": 0.2012641340494156,
      "reward_std": 0.9129719872027635,
      "rewards/reward_func": 0.2012641340494156,
      "step": 9030,
      "toxic_reward": 4.130698096752167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.85,
      "epoch": 2.1361058601134215,
      "format_reward": 0.0,
      "grad_norm": 6.688182830810547,
      "image_reward": 0.27446441650390624,
      "kl": 3.037346550822258,
      "learning_rate": 5e-06,
      "loss": -0.0086,
      "reward": 0.7237348094582557,
      "reward_std": 0.9079257231205702,
      "rewards/reward_func": 0.7237348094582557,
      "step": 9040,
      "toxic_reward": 4.051010203361511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.55,
      "epoch": 2.138468809073724,
      "format_reward": -0.25,
      "grad_norm": 2.8426876068115234,
      "image_reward": 0.26009623110294344,
      "kl": 234.93744373321533,
      "learning_rate": 5e-06,
      "loss": 0.0741,
      "reward": 0.14779042601585388,
      "reward_std": 1.6020304949954152,
      "rewards/reward_func": 0.14779042601585388,
      "step": 9050,
      "toxic_reward": 4.297617936134339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.55,
      "epoch": 2.1408317580340266,
      "format_reward": 0.0,
      "grad_norm": 14.305941581726074,
      "image_reward": 0.23194732666015624,
      "kl": 1.907991024851799,
      "learning_rate": 5e-06,
      "loss": -0.0095,
      "reward": 0.5170632779598237,
      "reward_std": 0.8214797399006784,
      "rewards/reward_func": 0.5170632779598237,
      "step": 9060,
      "toxic_reward": 4.2402391791343685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.0,
      "epoch": 2.143194706994329,
      "format_reward": 0.0,
      "grad_norm": 4.640130043029785,
      "image_reward": 0.2427520751953125,
      "kl": 7.810242688655853,
      "learning_rate": 5e-06,
      "loss": -0.032,
      "reward": 0.20102212131023406,
      "reward_std": 1.3208093732595443,
      "rewards/reward_func": 0.20102212131023406,
      "step": 9070,
      "toxic_reward": 3.7083510875701906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.1,
      "epoch": 2.145557655954631,
      "format_reward": 0.0,
      "grad_norm": 1.3630574941635132,
      "image_reward": 0.2402252197265625,
      "kl": 10.548876631259919,
      "learning_rate": 5e-06,
      "loss": 0.0112,
      "reward": 0.7412046194076538,
      "reward_std": 0.9147299766540528,
      "rewards/reward_func": 0.7412046194076538,
      "step": 9080,
      "toxic_reward": 4.000400519371032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.2,
      "epoch": 2.1479206049149338,
      "format_reward": 0.0,
      "grad_norm": 2.9857964515686035,
      "image_reward": 0.2373077392578125,
      "kl": 11.153948432207107,
      "learning_rate": 5e-06,
      "loss": -0.0525,
      "reward": 0.4043663561344147,
      "reward_std": 0.7483500481583178,
      "rewards/reward_func": 0.4043663561344147,
      "step": 9090,
      "toxic_reward": 4.85220890045166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.05,
      "epoch": 2.1502835538752363,
      "format_reward": 0.0,
      "grad_norm": 2.3352339267730713,
      "image_reward": 0.2479736328125,
      "kl": 3.4772801220417024,
      "learning_rate": 5e-06,
      "loss": 0.0421,
      "reward": 0.5415297746658325,
      "reward_std": 0.90726547986269,
      "rewards/reward_func": 0.5415297746658325,
      "step": 9100,
      "toxic_reward": 4.584239768981933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.125,
      "epoch": 2.152646502835539,
      "format_reward": 0.0,
      "grad_norm": 7.961544990539551,
      "image_reward": 0.2632598876953125,
      "kl": 2.5744317561388015,
      "learning_rate": 5e-06,
      "loss": 0.1083,
      "reward": 0.5891210317611695,
      "reward_std": 1.2839626222848892,
      "rewards/reward_func": 0.5891210317611695,
      "step": 9110,
      "toxic_reward": 4.181219959259034
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.075,
      "epoch": 2.1550094517958414,
      "format_reward": 0.0,
      "grad_norm": 9.345954895019531,
      "image_reward": 0.238812255859375,
      "kl": 6.054638743400574,
      "learning_rate": 5e-06,
      "loss": -0.1109,
      "reward": 0.06543984264135361,
      "reward_std": 0.519540898501873,
      "rewards/reward_func": 0.06543984264135361,
      "step": 9120,
      "toxic_reward": 4.270166897773743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.875,
      "epoch": 2.1573724007561434,
      "format_reward": 0.0,
      "grad_norm": 8.313584327697754,
      "image_reward": 0.24713134765625,
      "kl": 6.326075008511543,
      "learning_rate": 5e-06,
      "loss": -0.0767,
      "reward": 0.8043414294719696,
      "reward_std": 1.0881578013300897,
      "rewards/reward_func": 0.8043414294719696,
      "step": 9130,
      "toxic_reward": 4.155464768409729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.075,
      "epoch": 2.159735349716446,
      "format_reward": 0.0,
      "grad_norm": 8.283196449279785,
      "image_reward": 0.2358612060546875,
      "kl": 2.606276285648346,
      "learning_rate": 5e-06,
      "loss": 0.1205,
      "reward": 0.1571010023355484,
      "reward_std": 0.7534957839176059,
      "rewards/reward_func": 0.1571010023355484,
      "step": 9140,
      "toxic_reward": 4.338043940067291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.65,
      "epoch": 2.1620982986767485,
      "format_reward": 0.0,
      "grad_norm": 12.468461036682129,
      "image_reward": 0.2525299072265625,
      "kl": 10.651741808652877,
      "learning_rate": 5e-06,
      "loss": 0.0658,
      "reward": 0.499350106716156,
      "reward_std": 0.7922366757877171,
      "rewards/reward_func": 0.499350106716156,
      "step": 9150,
      "toxic_reward": 4.407905173301697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.5,
      "epoch": 2.164461247637051,
      "format_reward": 0.0,
      "grad_norm": 1.4810751676559448,
      "image_reward": 0.22776641845703124,
      "kl": 6.478110730648041,
      "learning_rate": 5e-06,
      "loss": -0.0464,
      "reward": 0.9298590540885925,
      "reward_std": 0.8800611793994904,
      "rewards/reward_func": 0.9298590540885925,
      "step": 9160,
      "toxic_reward": 4.576443719863891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.875,
      "epoch": 2.1668241965973536,
      "format_reward": 0.0,
      "grad_norm": 9.652629852294922,
      "image_reward": 0.260015869140625,
      "kl": 1.726886612176895,
      "learning_rate": 5e-06,
      "loss": -0.127,
      "reward": 0.4531150579452515,
      "reward_std": 0.8976183220744133,
      "rewards/reward_func": 0.4531150579452515,
      "step": 9170,
      "toxic_reward": 4.432527303695679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.0,
      "epoch": 2.169187145557656,
      "format_reward": 0.0,
      "grad_norm": 3.5050952434539795,
      "image_reward": 0.2384307861328125,
      "kl": 4.27691433429718,
      "learning_rate": 5e-06,
      "loss": -0.0311,
      "reward": 0.6826965510845184,
      "reward_std": 0.7877496212720871,
      "rewards/reward_func": 0.6826965510845184,
      "step": 9180,
      "toxic_reward": 4.4837501525878904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 52.1,
      "epoch": 2.171550094517958,
      "format_reward": 0.0,
      "grad_norm": 3.2137303352355957,
      "image_reward": 0.24414825439453125,
      "kl": 3.9786434292793276,
      "learning_rate": 5e-06,
      "loss": -0.048,
      "reward": 0.34388454258441925,
      "reward_std": 0.7642363490536809,
      "rewards/reward_func": 0.34388454258441925,
      "step": 9190,
      "toxic_reward": 4.380149924755097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.8,
      "epoch": 2.1739130434782608,
      "format_reward": 0.0,
      "grad_norm": 10.09927749633789,
      "image_reward": 0.2380889892578125,
      "kl": 3.893726623058319,
      "learning_rate": 5e-06,
      "loss": 0.0829,
      "reward": 0.28469178080558777,
      "reward_std": 0.7502976493909955,
      "rewards/reward_func": 0.28469178080558777,
      "step": 9200,
      "toxic_reward": 4.405940270423889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.25,
      "epoch": 2.1762759924385633,
      "format_reward": -0.25,
      "grad_norm": 36.55522537231445,
      "image_reward": 0.23165105208754538,
      "kl": 25.98146269917488,
      "learning_rate": 5e-06,
      "loss": 0.0734,
      "reward": -0.05637494325637817,
      "reward_std": 1.5777033947408199,
      "rewards/reward_func": -0.05637494325637817,
      "step": 9210,
      "toxic_reward": 4.138173961639405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.45,
      "epoch": 2.178638941398866,
      "format_reward": 0.0,
      "grad_norm": 2.0444726943969727,
      "image_reward": 0.2312713623046875,
      "kl": 2.639026927947998,
      "learning_rate": 5e-06,
      "loss": -0.0598,
      "reward": 0.5707060933113098,
      "reward_std": 1.1906714523211122,
      "rewards/reward_func": 0.5707060933113098,
      "step": 9220,
      "toxic_reward": 3.9467220425605776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.725,
      "epoch": 2.1810018903591684,
      "format_reward": 0.0,
      "grad_norm": 14.821864128112793,
      "image_reward": 0.260614013671875,
      "kl": 2.6187705636024474,
      "learning_rate": 5e-06,
      "loss": -0.0004,
      "reward": 0.776735657453537,
      "reward_std": 0.8302984148263931,
      "rewards/reward_func": 0.776735657453537,
      "step": 9230,
      "toxic_reward": 4.346326851844788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.95,
      "epoch": 2.183364839319471,
      "format_reward": 0.0,
      "grad_norm": 8.473363876342773,
      "image_reward": 0.227203369140625,
      "kl": 1.9701256573200225,
      "learning_rate": 5e-06,
      "loss": 0.0091,
      "reward": 0.6466103255748749,
      "reward_std": 0.5622012199833989,
      "rewards/reward_func": 0.6466103255748749,
      "step": 9240,
      "toxic_reward": 4.649976348876953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.9,
      "epoch": 2.185727788279773,
      "format_reward": 0.0,
      "grad_norm": 16.42177391052246,
      "image_reward": 0.2678741455078125,
      "kl": 3.791227114200592,
      "learning_rate": 5e-06,
      "loss": 0.0161,
      "reward": 0.5803273111581803,
      "reward_std": 0.8187548790127039,
      "rewards/reward_func": 0.5803273111581803,
      "step": 9250,
      "toxic_reward": 4.081698262691498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.5,
      "epoch": 2.1880907372400755,
      "format_reward": 0.0,
      "grad_norm": 23.73421859741211,
      "image_reward": 0.227972412109375,
      "kl": 3.5688813447952272,
      "learning_rate": 5e-06,
      "loss": 0.052,
      "reward": 0.5074087619781494,
      "reward_std": 0.9598018784075976,
      "rewards/reward_func": 0.5074087619781494,
      "step": 9260,
      "toxic_reward": 4.401837420463562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 60.425,
      "epoch": 2.190453686200378,
      "format_reward": 0.0,
      "grad_norm": 9.969075202941895,
      "image_reward": 0.245391845703125,
      "kl": 2.1753955483436584,
      "learning_rate": 5e-06,
      "loss": -0.0984,
      "reward": 0.9439354777336121,
      "reward_std": 0.7434614159166812,
      "rewards/reward_func": 0.9439354777336121,
      "step": 9270,
      "toxic_reward": 4.398157930374145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.325,
      "epoch": 2.1928166351606806,
      "format_reward": 0.0,
      "grad_norm": 11.830771446228027,
      "image_reward": 0.23084869384765624,
      "kl": 1.2656208366155624,
      "learning_rate": 5e-06,
      "loss": 0.0047,
      "reward": 0.31412020325660706,
      "reward_std": 0.8172964336816222,
      "rewards/reward_func": 0.31412020325660706,
      "step": 9280,
      "toxic_reward": 4.430534148216248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.475,
      "epoch": 2.195179584120983,
      "format_reward": 0.0,
      "grad_norm": 5.753904819488525,
      "image_reward": 0.24887237548828126,
      "kl": 1.3502902746200562,
      "learning_rate": 5e-06,
      "loss": 0.0623,
      "reward": 0.7708447635173797,
      "reward_std": 0.9731228679418564,
      "rewards/reward_func": 0.7708447635173797,
      "step": 9290,
      "toxic_reward": 4.273838710784912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.45,
      "epoch": 2.197542533081285,
      "format_reward": 0.0,
      "grad_norm": 7.7253947257995605,
      "image_reward": 0.2331573486328125,
      "kl": 3.059584191441536,
      "learning_rate": 5e-06,
      "loss": 0.0942,
      "reward": 0.43726455271244047,
      "reward_std": 1.3362455716356636,
      "rewards/reward_func": 0.43726455271244047,
      "step": 9300,
      "toxic_reward": 4.275756049156189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.525,
      "epoch": 2.1999054820415878,
      "format_reward": 0.0,
      "grad_norm": 14.917332649230957,
      "image_reward": 0.2485260009765625,
      "kl": 3.154623621702194,
      "learning_rate": 5e-06,
      "loss": -0.0028,
      "reward": 0.7619877219200134,
      "reward_std": 0.7480236226692796,
      "rewards/reward_func": 0.7619877219200134,
      "step": 9310,
      "toxic_reward": 4.5632892370224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.425,
      "epoch": 2.2022684310018903,
      "format_reward": 0.0,
      "grad_norm": 2.2232089042663574,
      "image_reward": 0.272552490234375,
      "kl": 4.570386919379234,
      "learning_rate": 5e-06,
      "loss": 0.0767,
      "reward": -0.32990662753582,
      "reward_std": 0.6452065747231245,
      "rewards/reward_func": -0.32990662753582,
      "step": 9320,
      "toxic_reward": 4.057078433036804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.475,
      "epoch": 2.204631379962193,
      "format_reward": 0.0,
      "grad_norm": 12.401063919067383,
      "image_reward": 0.22475687563419341,
      "kl": 2.3822293996810915,
      "learning_rate": 5e-06,
      "loss": -0.1368,
      "reward": 0.6720861852169037,
      "reward_std": 0.5467347849160433,
      "rewards/reward_func": 0.6720861852169037,
      "step": 9330,
      "toxic_reward": 4.748937749862671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.725,
      "epoch": 2.2069943289224954,
      "format_reward": 0.0,
      "grad_norm": 18.317251205444336,
      "image_reward": 0.2457794189453125,
      "kl": 2.1576267421245574,
      "learning_rate": 5e-06,
      "loss": 0.0443,
      "reward": 0.4097582340240479,
      "reward_std": 0.6367886804975569,
      "rewards/reward_func": 0.4097582340240479,
      "step": 9340,
      "toxic_reward": 3.8079322576522827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.0,
      "epoch": 2.209357277882798,
      "format_reward": 0.0,
      "grad_norm": 9.418509483337402,
      "image_reward": 0.2345733642578125,
      "kl": 0.8293094992637634,
      "learning_rate": 5e-06,
      "loss": 0.1013,
      "reward": 0.4199859380722046,
      "reward_std": 0.8710890758782626,
      "rewards/reward_func": 0.4199859380722046,
      "step": 9350,
      "toxic_reward": 4.7352869510650635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.7,
      "epoch": 2.2117202268431,
      "format_reward": 0.0,
      "grad_norm": 16.66223907470703,
      "image_reward": 0.25906219482421877,
      "kl": 1.3076835095882415,
      "learning_rate": 5e-06,
      "loss": 0.1376,
      "reward": 0.2918867290019989,
      "reward_std": 0.2911624666303396,
      "rewards/reward_func": 0.2918867290019989,
      "step": 9360,
      "toxic_reward": 4.793551731109619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.2,
      "epoch": 2.2140831758034025,
      "format_reward": 0.0,
      "grad_norm": 6.431090354919434,
      "image_reward": 0.214971923828125,
      "kl": 2.7103491842746736,
      "learning_rate": 5e-06,
      "loss": 0.0478,
      "reward": 0.3785900384187698,
      "reward_std": 0.8829892821609974,
      "rewards/reward_func": 0.3785900384187698,
      "step": 9370,
      "toxic_reward": 4.103424906730652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.45,
      "epoch": 2.216446124763705,
      "format_reward": 0.0,
      "grad_norm": 5.423946857452393,
      "image_reward": 0.2241668701171875,
      "kl": 9.351680633425712,
      "learning_rate": 5e-06,
      "loss": -0.0282,
      "reward": 0.6931559234857559,
      "reward_std": 0.9827461183071137,
      "rewards/reward_func": 0.6931559234857559,
      "step": 9380,
      "toxic_reward": 4.327680516242981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.75,
      "epoch": 2.2188090737240076,
      "format_reward": 0.0,
      "grad_norm": 12.53814697265625,
      "image_reward": 0.24124603271484374,
      "kl": 1.539423054456711,
      "learning_rate": 5e-06,
      "loss": 0.0074,
      "reward": 0.5662744238972663,
      "reward_std": 0.8970771560445427,
      "rewards/reward_func": 0.5662744238972663,
      "step": 9390,
      "toxic_reward": 3.676301693916321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.85,
      "epoch": 2.22117202268431,
      "format_reward": -0.25,
      "grad_norm": 27.224220275878906,
      "image_reward": 0.26278177797794344,
      "kl": 8.620309627056121,
      "learning_rate": 5e-06,
      "loss": -0.0561,
      "reward": 0.45245649218559264,
      "reward_std": 1.51493993550539,
      "rewards/reward_func": 0.45245649218559264,
      "step": 9400,
      "toxic_reward": 4.195838165283203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.3,
      "epoch": 2.2235349716446127,
      "format_reward": 0.0,
      "grad_norm": 16.83915901184082,
      "image_reward": 0.244732666015625,
      "kl": 7.121062386035919,
      "learning_rate": 5e-06,
      "loss": 0.0277,
      "reward": 0.5056580305099487,
      "reward_std": 0.6380140800029039,
      "rewards/reward_func": 0.5056580305099487,
      "step": 9410,
      "toxic_reward": 4.542606806755066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.525,
      "epoch": 2.2258979206049148,
      "format_reward": 0.0,
      "grad_norm": 14.892727851867676,
      "image_reward": 0.24930419921875,
      "kl": 5.096332561969757,
      "learning_rate": 5e-06,
      "loss": -0.0329,
      "reward": 0.48427205085754393,
      "reward_std": 1.0285473830997944,
      "rewards/reward_func": 0.48427205085754393,
      "step": 9420,
      "toxic_reward": 4.446974515914917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.75,
      "epoch": 2.2282608695652173,
      "format_reward": -0.25,
      "grad_norm": 1.2352709770202637,
      "image_reward": 0.24373575747013093,
      "kl": 16.825757110118865,
      "learning_rate": 5e-06,
      "loss": -0.0367,
      "reward": 0.5291045546531677,
      "reward_std": 1.2605504954233766,
      "rewards/reward_func": 0.5291045546531677,
      "step": 9430,
      "toxic_reward": 4.275347375869751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.875,
      "epoch": 2.23062381852552,
      "format_reward": 0.0,
      "grad_norm": 22.900882720947266,
      "image_reward": 0.25095672607421876,
      "kl": 10.578787690401077,
      "learning_rate": 5e-06,
      "loss": 0.0032,
      "reward": 0.26428125500679017,
      "reward_std": 0.9156784310936927,
      "rewards/reward_func": 0.26428125500679017,
      "step": 9440,
      "toxic_reward": 3.866254734992981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.5,
      "epoch": 2.2329867674858224,
      "format_reward": -0.25,
      "grad_norm": 13.677875518798828,
      "image_reward": 0.238702392578125,
      "kl": 5.4229684472084045,
      "learning_rate": 5e-06,
      "loss": -0.0035,
      "reward": 0.34162178039550783,
      "reward_std": 1.2561071523465217,
      "rewards/reward_func": 0.34162178039550783,
      "step": 9450,
      "toxic_reward": 4.54083218574524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.25,
      "epoch": 2.235349716446125,
      "format_reward": 0.0,
      "grad_norm": 7.939948081970215,
      "image_reward": 0.245355224609375,
      "kl": 2.677640450000763,
      "learning_rate": 5e-06,
      "loss": -0.0006,
      "reward": 0.28101458549499514,
      "reward_std": 0.8911348965018988,
      "rewards/reward_func": 0.28101458549499514,
      "step": 9460,
      "toxic_reward": 4.145281267166138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.0,
      "epoch": 2.237712665406427,
      "format_reward": 0.0,
      "grad_norm": 1.8240618705749512,
      "image_reward": 0.231634521484375,
      "kl": 1.528824520111084,
      "learning_rate": 5e-06,
      "loss": -0.0875,
      "reward": 0.37455313801765444,
      "reward_std": 0.4142075888812542,
      "rewards/reward_func": 0.37455313801765444,
      "step": 9470,
      "toxic_reward": 4.762905406951904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.325,
      "epoch": 2.2400756143667295,
      "format_reward": 0.0,
      "grad_norm": 6.417304515838623,
      "image_reward": 0.2490814208984375,
      "kl": 1.3931379437446594,
      "learning_rate": 5e-06,
      "loss": 0.0247,
      "reward": 1.1561188876628876,
      "reward_std": 0.7829106822609901,
      "rewards/reward_func": 1.1561188876628876,
      "step": 9480,
      "toxic_reward": 4.180247139930725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.1,
      "epoch": 2.242438563327032,
      "format_reward": 0.0,
      "grad_norm": 6.768500328063965,
      "image_reward": 0.2283935546875,
      "kl": 0.9197474420070648,
      "learning_rate": 5e-06,
      "loss": -0.0378,
      "reward": 0.5479820281267166,
      "reward_std": 0.8298372723162174,
      "rewards/reward_func": 0.5479820281267166,
      "step": 9490,
      "toxic_reward": 4.205300378799438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.375,
      "epoch": 2.2448015122873346,
      "format_reward": 0.0,
      "grad_norm": 26.945127487182617,
      "image_reward": 0.26320343017578124,
      "kl": 1.3565968126058578,
      "learning_rate": 5e-06,
      "loss": 0.0841,
      "reward": 0.7786126613616944,
      "reward_std": 1.1838067084550858,
      "rewards/reward_func": 0.7786126613616944,
      "step": 9500,
      "toxic_reward": 3.8664269924163817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 36.3,
      "epoch": 2.247164461247637,
      "format_reward": -0.25,
      "grad_norm": 5.876742839813232,
      "image_reward": 0.2632466644048691,
      "kl": 2.275825482606888,
      "learning_rate": 5e-06,
      "loss": -0.0704,
      "reward": 0.1731979250907898,
      "reward_std": 1.4120358280837535,
      "rewards/reward_func": 0.1731979250907898,
      "step": 9510,
      "toxic_reward": 4.367373514175415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.525,
      "epoch": 2.2495274102079397,
      "format_reward": 0.0,
      "grad_norm": 4.764988422393799,
      "image_reward": 0.2396881103515625,
      "kl": 5.604674518108368,
      "learning_rate": 5e-06,
      "loss": 0.0467,
      "reward": 0.2715915977954865,
      "reward_std": 0.6032503295689822,
      "rewards/reward_func": 0.2715915977954865,
      "step": 9520,
      "toxic_reward": 4.3036177396774296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.15,
      "epoch": 2.251890359168242,
      "format_reward": 0.0,
      "grad_norm": 23.23556900024414,
      "image_reward": 0.2485931396484375,
      "kl": 2.390829586982727,
      "learning_rate": 5e-06,
      "loss": 0.0824,
      "reward": 0.7335005760192871,
      "reward_std": 0.5389063037931919,
      "rewards/reward_func": 0.7335005760192871,
      "step": 9530,
      "toxic_reward": 4.7080058574676515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.55,
      "epoch": 2.2542533081285443,
      "format_reward": 0.0,
      "grad_norm": 20.620105743408203,
      "image_reward": 0.2229095458984375,
      "kl": 5.458765661716461,
      "learning_rate": 5e-06,
      "loss": 0.1314,
      "reward": 0.5191788256168366,
      "reward_std": 1.1402710743248463,
      "rewards/reward_func": 0.5191788256168366,
      "step": 9540,
      "toxic_reward": 4.577926540374756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.75,
      "epoch": 2.256616257088847,
      "format_reward": 0.0,
      "grad_norm": 6.686956405639648,
      "image_reward": 0.231097412109375,
      "kl": 2.384101688861847,
      "learning_rate": 5e-06,
      "loss": -0.0264,
      "reward": 0.3123217046260834,
      "reward_std": 1.1325789090245961,
      "rewards/reward_func": 0.3123217046260834,
      "step": 9550,
      "toxic_reward": 4.343110990524292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.8,
      "epoch": 2.2589792060491494,
      "format_reward": 0.0,
      "grad_norm": 11.559769630432129,
      "image_reward": 0.236077880859375,
      "kl": 2.987881660461426,
      "learning_rate": 5e-06,
      "loss": 0.0371,
      "reward": 1.1903966188430786,
      "reward_std": 0.8922829747200012,
      "rewards/reward_func": 1.1903966188430786,
      "step": 9560,
      "toxic_reward": 4.265221381187439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.15,
      "epoch": 2.261342155009452,
      "format_reward": -0.25,
      "grad_norm": 8.45358657836914,
      "image_reward": 0.2397247314453125,
      "kl": 3.8612433314323424,
      "learning_rate": 5e-06,
      "loss": -0.0161,
      "reward": -0.07919068932533264,
      "reward_std": 1.113222143240273,
      "rewards/reward_func": -0.07919068932533264,
      "step": 9570,
      "toxic_reward": 4.036798477172852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.95,
      "epoch": 2.2637051039697544,
      "format_reward": 0.0,
      "grad_norm": 9.02270221710205,
      "image_reward": 0.232049560546875,
      "kl": 4.92200248837471,
      "learning_rate": 5e-06,
      "loss": 0.0855,
      "reward": 0.7455990195274353,
      "reward_std": 0.8981746949255467,
      "rewards/reward_func": 0.7455990195274353,
      "step": 9580,
      "toxic_reward": 4.544493341445923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.2,
      "epoch": 2.2660680529300565,
      "format_reward": 0.0,
      "grad_norm": 19.980321884155273,
      "image_reward": 0.2468475341796875,
      "kl": 3.3157293617725374,
      "learning_rate": 5e-06,
      "loss": 0.1177,
      "reward": 0.4287958800792694,
      "reward_std": 0.8264384102076292,
      "rewards/reward_func": 0.4287958800792694,
      "step": 9590,
      "toxic_reward": 4.492921185493469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.425,
      "epoch": 2.268431001890359,
      "format_reward": 0.0,
      "grad_norm": 1.6929293870925903,
      "image_reward": 0.2279388427734375,
      "kl": 11.748549377918243,
      "learning_rate": 5e-06,
      "loss": 0.0227,
      "reward": 0.8450492799282074,
      "reward_std": 0.8821253469213843,
      "rewards/reward_func": 0.8450492799282074,
      "step": 9600,
      "toxic_reward": 4.329080724716187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.075,
      "epoch": 2.2707939508506616,
      "format_reward": -0.25,
      "grad_norm": 6.6792683601379395,
      "image_reward": 0.2432342529296875,
      "kl": 23.715504467487335,
      "learning_rate": 5e-06,
      "loss": -0.0603,
      "reward": 0.6058377146720886,
      "reward_std": 1.8178761571645736,
      "rewards/reward_func": 0.6058377146720886,
      "step": 9610,
      "toxic_reward": 3.8193166494369506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.6,
      "epoch": 2.273156899810964,
      "format_reward": -0.25,
      "grad_norm": 4.806612014770508,
      "image_reward": 0.25201212614774704,
      "kl": 2.9662541508674622,
      "learning_rate": 5e-06,
      "loss": 0.0205,
      "reward": 0.14691731929779053,
      "reward_std": 1.3215140633285045,
      "rewards/reward_func": 0.14691731929779053,
      "step": 9620,
      "toxic_reward": 4.118840670585632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.875,
      "epoch": 2.2755198487712667,
      "format_reward": 0.0,
      "grad_norm": 11.628203392028809,
      "image_reward": 0.2614166259765625,
      "kl": 3.4263065993785857,
      "learning_rate": 5e-06,
      "loss": 0.0042,
      "reward": 0.7031525075435638,
      "reward_std": 0.9792011518031358,
      "rewards/reward_func": 0.7031525075435638,
      "step": 9630,
      "toxic_reward": 4.620864820480347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 33.3,
      "epoch": 2.2778827977315688,
      "format_reward": 0.0,
      "grad_norm": 10.783760070800781,
      "image_reward": 0.2508331298828125,
      "kl": 12.492328238487243,
      "learning_rate": 5e-06,
      "loss": 0.0118,
      "reward": 0.2551820993423462,
      "reward_std": 0.8312258010730147,
      "rewards/reward_func": 0.2551820993423462,
      "step": 9640,
      "toxic_reward": 4.169378912448883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.475,
      "epoch": 2.2802457466918713,
      "format_reward": 0.0,
      "grad_norm": 9.430181503295898,
      "image_reward": 0.256829833984375,
      "kl": 10.857812678813934,
      "learning_rate": 5e-06,
      "loss": 0.0124,
      "reward": 0.6474542915821075,
      "reward_std": 1.0158755726995878,
      "rewards/reward_func": 0.6474542915821075,
      "step": 9650,
      "toxic_reward": 4.2302504777908325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.225,
      "epoch": 2.282608695652174,
      "format_reward": 0.0,
      "grad_norm": 2.653324604034424,
      "image_reward": 0.258251953125,
      "kl": 4.948359310626984,
      "learning_rate": 5e-06,
      "loss": -0.0303,
      "reward": 0.8631646454334259,
      "reward_std": 1.4457193814218043,
      "rewards/reward_func": 0.8631646454334259,
      "step": 9660,
      "toxic_reward": 4.028263640403748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.475,
      "epoch": 2.2849716446124764,
      "format_reward": 0.0,
      "grad_norm": 46.17441940307617,
      "image_reward": 0.2468775436282158,
      "kl": 4.73446731865406,
      "learning_rate": 5e-06,
      "loss": -0.0188,
      "reward": 0.47216950058937074,
      "reward_std": 0.5647860389202833,
      "rewards/reward_func": 0.47216950058937074,
      "step": 9670,
      "toxic_reward": 4.644181919097901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.975,
      "epoch": 2.287334593572779,
      "format_reward": 0.0,
      "grad_norm": 6.452030181884766,
      "image_reward": 0.24481913298368455,
      "kl": 84.93760406374932,
      "learning_rate": 5e-06,
      "loss": -0.0133,
      "reward": 0.45135449171066283,
      "reward_std": 0.9933142360066995,
      "rewards/reward_func": 0.45135449171066283,
      "step": 9680,
      "toxic_reward": 4.119644379615783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 50.925,
      "epoch": 2.2896975425330814,
      "format_reward": 0.0,
      "grad_norm": 7.41399621963501,
      "image_reward": 0.2643157958984375,
      "kl": 2.3054057717323304,
      "learning_rate": 5e-06,
      "loss": 0.0556,
      "reward": 0.8660075426101684,
      "reward_std": 1.1269529208540916,
      "rewards/reward_func": 0.8660075426101684,
      "step": 9690,
      "toxic_reward": 4.305045056343078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 56.95,
      "epoch": 2.292060491493384,
      "format_reward": 0.0,
      "grad_norm": 4.1870198249816895,
      "image_reward": 0.255328369140625,
      "kl": 0.9636234432458878,
      "learning_rate": 5e-06,
      "loss": 0.0444,
      "reward": 0.07925584316253662,
      "reward_std": 0.9312605137005449,
      "rewards/reward_func": 0.07925584316253662,
      "step": 9700,
      "toxic_reward": 4.480338978767395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.175,
      "epoch": 2.294423440453686,
      "format_reward": 0.0,
      "grad_norm": 17.8924617767334,
      "image_reward": 0.2297576904296875,
      "kl": 1.2910286843776704,
      "learning_rate": 5e-06,
      "loss": -0.0594,
      "reward": 0.09658912420272828,
      "reward_std": 0.534552292432636,
      "rewards/reward_func": 0.09658912420272828,
      "step": 9710,
      "toxic_reward": 4.421128726005554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.5,
      "epoch": 2.2967863894139886,
      "format_reward": 0.0,
      "grad_norm": 8.845993995666504,
      "image_reward": 0.260223388671875,
      "kl": 11.969202554225921,
      "learning_rate": 5e-06,
      "loss": 0.0984,
      "reward": 0.49135610461235046,
      "reward_std": 1.074414287507534,
      "rewards/reward_func": 0.49135610461235046,
      "step": 9720,
      "toxic_reward": 4.317939972877502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 54.725,
      "epoch": 2.299149338374291,
      "format_reward": 0.0,
      "grad_norm": 2.5245182514190674,
      "image_reward": 0.2630401611328125,
      "kl": 18.936833548545838,
      "learning_rate": 5e-06,
      "loss": 0.0013,
      "reward": 0.10787631869316101,
      "reward_std": 0.7801851622760296,
      "rewards/reward_func": 0.10787631869316101,
      "step": 9730,
      "toxic_reward": 4.423897337913513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 47.875,
      "epoch": 2.3015122873345937,
      "format_reward": -0.25,
      "grad_norm": 3.0716590881347656,
      "image_reward": 0.25508524626493456,
      "kl": 2.160058504343033,
      "learning_rate": 5e-06,
      "loss": -0.0232,
      "reward": 0.1063625156879425,
      "reward_std": 1.0759605418890714,
      "rewards/reward_func": 0.1063625156879425,
      "step": 9740,
      "toxic_reward": 4.3748561382293705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.475,
      "epoch": 2.303875236294896,
      "format_reward": 0.0,
      "grad_norm": 2.622436285018921,
      "image_reward": 0.23123779296875,
      "kl": 8.901539516448974,
      "learning_rate": 5e-06,
      "loss": 0.0451,
      "reward": 0.41706631779670716,
      "reward_std": 0.8451563934795558,
      "rewards/reward_func": 0.41706631779670716,
      "step": 9750,
      "toxic_reward": 4.452003169059753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.425,
      "epoch": 2.3062381852551983,
      "format_reward": 0.0,
      "grad_norm": 7.1622514724731445,
      "image_reward": 0.2591522216796875,
      "kl": 17.092305302619934,
      "learning_rate": 5e-06,
      "loss": 0.0045,
      "reward": 0.43682674169540403,
      "reward_std": 1.0151184625923633,
      "rewards/reward_func": 0.43682674169540403,
      "step": 9760,
      "toxic_reward": 3.4172345459461213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.35,
      "epoch": 2.308601134215501,
      "format_reward": 0.0,
      "grad_norm": 12.564181327819824,
      "image_reward": 0.2551788330078125,
      "kl": 8.035814380645752,
      "learning_rate": 5e-06,
      "loss": -0.057,
      "reward": 0.47375474870204926,
      "reward_std": 0.6293097786605358,
      "rewards/reward_func": 0.47375474870204926,
      "step": 9770,
      "toxic_reward": 3.9829455375671388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.425,
      "epoch": 2.3109640831758034,
      "format_reward": 0.0,
      "grad_norm": 12.808835983276367,
      "image_reward": 0.2319427490234375,
      "kl": 3.5698849081993105,
      "learning_rate": 5e-06,
      "loss": 0.0338,
      "reward": 0.6212572991847992,
      "reward_std": 0.6545703388750553,
      "rewards/reward_func": 0.6212572991847992,
      "step": 9780,
      "toxic_reward": 4.615470266342163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.7,
      "epoch": 2.313327032136106,
      "format_reward": 0.0,
      "grad_norm": 11.370565414428711,
      "image_reward": 0.2495208740234375,
      "kl": 58.05081114768982,
      "learning_rate": 5e-06,
      "loss": -0.003,
      "reward": 0.3863606512546539,
      "reward_std": 0.7871608097106219,
      "rewards/reward_func": 0.3863606512546539,
      "step": 9790,
      "toxic_reward": 4.543632507324219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 53.225,
      "epoch": 2.3156899810964084,
      "format_reward": 0.0,
      "grad_norm": 17.273242950439453,
      "image_reward": 0.256707763671875,
      "kl": 63.41283442378044,
      "learning_rate": 5e-06,
      "loss": -0.0867,
      "reward": -0.17932948917150499,
      "reward_std": 0.5697868175804615,
      "rewards/reward_func": -0.17932948917150499,
      "step": 9800,
      "toxic_reward": 3.8667405366897585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 46.575,
      "epoch": 2.3180529300567105,
      "format_reward": 0.0,
      "grad_norm": 2.3364224433898926,
      "image_reward": 0.272943115234375,
      "kl": 2.4312843918800353,
      "learning_rate": 5e-06,
      "loss": 0.0565,
      "reward": 0.25352796316146853,
      "reward_std": 0.8136134160682559,
      "rewards/reward_func": 0.25352796316146853,
      "step": 9810,
      "toxic_reward": 4.419120264053345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.975,
      "epoch": 2.320415879017013,
      "format_reward": 0.0,
      "grad_norm": 16.5479793548584,
      "image_reward": 0.2720245361328125,
      "kl": 4.018320089578628,
      "learning_rate": 5e-06,
      "loss": -0.052,
      "reward": 0.3462803453207016,
      "reward_std": 0.6793602051213383,
      "rewards/reward_func": 0.3462803453207016,
      "step": 9820,
      "toxic_reward": 4.1594162940979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.5,
      "epoch": 2.3227788279773156,
      "format_reward": 0.0,
      "grad_norm": 16.928800582885742,
      "image_reward": 0.2370941162109375,
      "kl": 5.093863940238952,
      "learning_rate": 5e-06,
      "loss": -0.0256,
      "reward": -0.17395999431610107,
      "reward_std": 0.733401482924819,
      "rewards/reward_func": -0.17395999431610107,
      "step": 9830,
      "toxic_reward": 4.316698336601258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 40.3,
      "epoch": 2.325141776937618,
      "format_reward": 0.0,
      "grad_norm": 5.420310020446777,
      "image_reward": 0.2193939208984375,
      "kl": 10.776195186376572,
      "learning_rate": 5e-06,
      "loss": 0.0166,
      "reward": 0.9508269459009171,
      "reward_std": 1.237346090376377,
      "rewards/reward_func": 0.9508269459009171,
      "step": 9840,
      "toxic_reward": 4.61365122795105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 43.6,
      "epoch": 2.3275047258979207,
      "format_reward": 0.0,
      "grad_norm": 8.147756576538086,
      "image_reward": 0.2236907958984375,
      "kl": 14.836266088485718,
      "learning_rate": 5e-06,
      "loss": -0.0104,
      "reward": 1.2669459402561187,
      "reward_std": 0.8822499677538872,
      "rewards/reward_func": 1.2669459402561187,
      "step": 9850,
      "toxic_reward": 4.57221360206604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 51.225,
      "epoch": 2.329867674858223,
      "format_reward": 0.0,
      "grad_norm": 9.47561264038086,
      "image_reward": 0.2329345703125,
      "kl": 7.03211784362793,
      "learning_rate": 5e-06,
      "loss": 0.1103,
      "reward": 0.4148245692253113,
      "reward_std": 0.4656851476058364,
      "rewards/reward_func": 0.4148245692253113,
      "step": 9860,
      "toxic_reward": 4.482611513137817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.45,
      "epoch": 2.3322306238185257,
      "format_reward": 0.0,
      "grad_norm": 5.070466995239258,
      "image_reward": 0.242041015625,
      "kl": 4.666208404302597,
      "learning_rate": 5e-06,
      "loss": -0.0147,
      "reward": 0.29748362898826597,
      "reward_std": 0.7133689053356648,
      "rewards/reward_func": 0.29748362898826597,
      "step": 9870,
      "toxic_reward": 4.607629799842835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 48.7,
      "epoch": 2.334593572778828,
      "format_reward": 0.0,
      "grad_norm": 18.918405532836914,
      "image_reward": 0.2441070556640625,
      "kl": 1.534792199730873,
      "learning_rate": 5e-06,
      "loss": -0.0478,
      "reward": 1.092936259508133,
      "reward_std": 1.04658992420882,
      "rewards/reward_func": 1.092936259508133,
      "step": 9880,
      "toxic_reward": 4.236094212532043
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.525,
      "epoch": 2.3369565217391304,
      "format_reward": -0.25,
      "grad_norm": 2.751826524734497,
      "image_reward": 0.239617919921875,
      "kl": 4.680880203843117,
      "learning_rate": 5e-06,
      "loss": 0.0816,
      "reward": 0.287129682302475,
      "reward_std": 1.596864845789969,
      "rewards/reward_func": 0.287129682302475,
      "step": 9890,
      "toxic_reward": 3.968970334529877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 37.05,
      "epoch": 2.339319470699433,
      "format_reward": 0.0,
      "grad_norm": 7.023763179779053,
      "image_reward": 0.22930908203125,
      "kl": 2.7311850488185883,
      "learning_rate": 5e-06,
      "loss": 0.0339,
      "reward": 0.4986346364021301,
      "reward_std": 1.2056658655405044,
      "rewards/reward_func": 0.4986346364021301,
      "step": 9900,
      "toxic_reward": 4.340516233444214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 41.425,
      "epoch": 2.3416824196597354,
      "format_reward": -0.5,
      "grad_norm": 10.157588958740234,
      "image_reward": 0.24829813539981843,
      "kl": 2.7676683485507967,
      "learning_rate": 5e-06,
      "loss": 0.0746,
      "reward": -0.3049712359905243,
      "reward_std": 1.7655695647001266,
      "rewards/reward_func": -0.3049712359905243,
      "step": 9910,
      "toxic_reward": 4.103359699249268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.525,
      "epoch": 2.344045368620038,
      "format_reward": -0.25,
      "grad_norm": 13.108144760131836,
      "image_reward": 0.22231547087430953,
      "kl": 0.8436797827482223,
      "learning_rate": 5e-06,
      "loss": -0.0158,
      "reward": 0.21243730187416077,
      "reward_std": 1.2439281724393367,
      "rewards/reward_func": 0.21243730187416077,
      "step": 9920,
      "toxic_reward": 4.508874106407165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 39.875,
      "epoch": 2.34640831758034,
      "format_reward": 0.0,
      "grad_norm": 7.745576858520508,
      "image_reward": 0.2392669677734375,
      "kl": 2.106270205974579,
      "learning_rate": 5e-06,
      "loss": 0.0696,
      "reward": 0.02986244559288025,
      "reward_std": 0.5784997101873159,
      "rewards/reward_func": 0.02986244559288025,
      "step": 9930,
      "toxic_reward": 4.542821955680847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.5,
      "epoch": 2.3487712665406426,
      "format_reward": 0.0,
      "grad_norm": 12.980175018310547,
      "image_reward": 0.26973876953125,
      "kl": 6.715492057800293,
      "learning_rate": 5e-06,
      "loss": -0.0504,
      "reward": 0.8542561173439026,
      "reward_std": 0.8472011580131948,
      "rewards/reward_func": 0.8542561173439026,
      "step": 9940,
      "toxic_reward": 4.496533703804016
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 45.775,
      "epoch": 2.351134215500945,
      "format_reward": 0.0,
      "grad_norm": 8.429265975952148,
      "image_reward": 0.245904541015625,
      "kl": 1.740691715478897,
      "learning_rate": 5e-06,
      "loss": 0.0812,
      "reward": 0.524560397863388,
      "reward_std": 0.6684761707670986,
      "rewards/reward_func": 0.524560397863388,
      "step": 9950,
      "toxic_reward": 4.557698893547058
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 49.725,
      "epoch": 2.3534971644612477,
      "format_reward": 0.0,
      "grad_norm": 7.811772346496582,
      "image_reward": 0.23507537841796874,
      "kl": 4.798752707242966,
      "learning_rate": 5e-06,
      "loss": 0.0502,
      "reward": 0.2622858464717865,
      "reward_std": 0.7538703501224517,
      "rewards/reward_func": 0.2622858464717865,
      "step": 9960,
      "toxic_reward": 4.338898825645447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 35.6,
      "epoch": 2.35586011342155,
      "format_reward": 0.0,
      "grad_norm": 3.807326555252075,
      "image_reward": 0.227703857421875,
      "kl": 0.7807338133454322,
      "learning_rate": 5e-06,
      "loss": 0.0267,
      "reward": 1.1823074579238892,
      "reward_std": 1.5363767087459563,
      "rewards/reward_func": 1.1823074579238892,
      "step": 9970,
      "toxic_reward": 4.390237951278687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 44.225,
      "epoch": 2.3582230623818523,
      "format_reward": 0.0,
      "grad_norm": 1.2711127996444702,
      "image_reward": 0.23590087890625,
      "kl": 1.0028429985046388,
      "learning_rate": 5e-06,
      "loss": -0.0221,
      "reward": 0.1531411349773407,
      "reward_std": 0.5583895549178124,
      "rewards/reward_func": 0.1531411349773407,
      "step": 9980,
      "toxic_reward": 4.677754878997803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 42.775,
      "epoch": 2.360586011342155,
      "format_reward": 0.0,
      "grad_norm": 2.5527610778808594,
      "image_reward": 0.24173583984375,
      "kl": 1.1347097665071488,
      "learning_rate": 5e-06,
      "loss": 0.0127,
      "reward": -0.07762867212295532,
      "reward_std": 0.6002773646265268,
      "rewards/reward_func": -0.07762867212295532,
      "step": 9990,
      "toxic_reward": 4.172585511207581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 38.05,
      "epoch": 2.3629489603024574,
      "format_reward": 0.0,
      "grad_norm": 4.715011119842529,
      "image_reward": 0.23904571533203126,
      "kl": 7.07305488884449,
      "learning_rate": 5e-06,
      "loss": -0.021,
      "reward": 0.26526654958724977,
      "reward_std": 0.82782434374094,
      "rewards/reward_func": 0.26526654958724977,
      "step": 10000,
      "toxic_reward": 4.33233335018158
    }
  ],
  "logging_steps": 10,
  "max_steps": 100000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 24,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}