trainer_state.json

{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.11428571428571428,
  "eval_steps": 500,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_fraction": 0.0,
      "completion_length": 2523.270866394043,
      "epoch": 0.001142857142857143,
      "grad_norm": 0.06767117232084274,
      "kl": 0.0,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 0.0,
      "loss": -0.0168,
      "reward": -0.06946920603513718,
      "reward_after_mean": -0.06946920603513718,
      "reward_after_std": 0.5871539004147053,
      "reward_before_mean": 0.17862090840935707,
      "reward_before_std": 0.5394803490489721,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2480901088565588,
      "reward_change_min": -0.44155892729759216,
      "reward_change_std": 0.16163883404806256,
      "reward_std": 0.5871539227664471,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/cosine_scaled_reward": -0.0713790925219655,
      "step": 1
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2684.583366394043,
      "epoch": 0.002285714285714286,
      "grad_norm": 0.07669047266244888,
      "kl": 0.0,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1e-07,
      "loss": 0.0043,
      "reward": 0.034156665205955505,
      "reward_after_mean": 0.034156665205955505,
      "reward_after_std": 0.46472928673028946,
      "reward_before_mean": 0.33918463438749313,
      "reward_before_std": 0.41114553064107895,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.30502799339592457,
      "reward_change_min": -0.46970658004283905,
      "reward_change_std": 0.17754351254552603,
      "reward_std": 0.46472930535674095,
      "rewards/accuracy_reward": 0.29166667349636555,
      "rewards/cosine_scaled_reward": 0.047517990693449974,
      "step": 2
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2952.6458587646484,
      "epoch": 0.0034285714285714284,
      "grad_norm": 0.08670737594366074,
      "kl": 5.030632019042969e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2e-07,
      "loss": -0.0823,
      "reward": -0.286656855372712,
      "reward_after_mean": -0.286656855372712,
      "reward_after_std": 0.3848829958587885,
      "reward_before_mean": -0.09138839645311236,
      "reward_before_std": 0.32367083616554737,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.1952684484422207,
      "reward_change_min": -0.2791023626923561,
      "reward_change_std": 0.1039092754945159,
      "reward_std": 0.38488300889730453,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/cosine_scaled_reward": -0.13305507553741336,
      "step": 3
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 1583.8958587646484,
      "epoch": 0.004571428571428572,
      "grad_norm": 0.1990806609392166,
      "kl": 3.854930400848389e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3e-07,
      "loss": -0.1819,
      "reward": -0.11896039673592895,
      "reward_after_mean": -0.11896039673592895,
      "reward_after_std": 0.5856046769768,
      "reward_before_mean": 0.11259861849248409,
      "reward_before_std": 0.5683185886591673,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.23155900090932846,
      "reward_change_min": -0.4133305959403515,
      "reward_change_std": 0.15685046929866076,
      "reward_std": 0.5856046807020903,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/cosine_scaled_reward": -0.05406807316467166,
      "step": 4
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3137.166717529297,
      "epoch": 0.005714285714285714,
      "grad_norm": 0.07480327039957047,
      "kl": 5.798041820526123e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 4e-07,
      "loss": 0.0034,
      "reward": -0.2642638385295868,
      "reward_after_mean": -0.2642638385295868,
      "reward_after_std": 0.47306079883128405,
      "reward_before_mean": -0.06776071339845657,
      "reward_before_std": 0.4494105405174196,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.19650312513113022,
      "reward_change_min": -0.39078205823898315,
      "reward_change_std": 0.13734673336148262,
      "reward_std": 0.47306080628186464,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.1719273824710399,
      "step": 5
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2746.5416717529297,
      "epoch": 0.006857142857142857,
      "grad_norm": 0.08584456145763397,
      "kl": 5.7503581047058105e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5e-07,
      "loss": 0.012,
      "reward": -0.2835393473505974,
      "reward_after_mean": -0.2835393473505974,
      "reward_after_std": 0.2985193133354187,
      "reward_before_mean": -0.06978187244385481,
      "reward_before_std": 0.27547394298017025,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.21375747956335545,
      "reward_change_min": -0.34073719196021557,
      "reward_change_std": 0.13080975599586964,
      "reward_std": 0.29851931519806385,
      "rewards/accuracy_reward": 0.0625,
      "rewards/cosine_scaled_reward": -0.13228187058120966,
      "step": 6
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2684.8334045410156,
      "epoch": 0.008,
      "grad_norm": 0.06891732662916183,
      "kl": 3.771483898162842e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 6e-07,
      "loss": -0.0261,
      "reward": -0.040946679189801216,
      "reward_after_mean": -0.040946679189801216,
      "reward_after_std": 0.4143037796020508,
      "reward_before_mean": 0.24522151239216328,
      "reward_before_std": 0.36546519538387656,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2861681915819645,
      "reward_change_min": -0.47350434213876724,
      "reward_change_std": 0.17670375108718872,
      "reward_std": 0.4143037870526314,
      "rewards/accuracy_reward": 0.2291666679084301,
      "rewards/cosine_scaled_reward": 0.016054846346378326,
      "step": 7
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2381.770881652832,
      "epoch": 0.009142857142857144,
      "grad_norm": 0.0757519081234932,
      "kl": 3.248453140258789e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7e-07,
      "loss": -0.0307,
      "reward": 0.11849129945039749,
      "reward_after_mean": 0.11849129945039749,
      "reward_after_std": 0.5733677167445421,
      "reward_before_mean": 0.44195539876818657,
      "reward_before_std": 0.5410135546699166,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.3234641030430794,
      "reward_change_min": -0.5369394198060036,
      "reward_change_std": 0.20503532141447067,
      "reward_std": 0.5733677390962839,
      "rewards/accuracy_reward": 0.35416667349636555,
      "rewards/cosine_scaled_reward": 0.08778871223330498,
      "step": 8
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2806.8125534057617,
      "epoch": 0.010285714285714285,
      "grad_norm": 0.10130124539136887,
      "kl": 5.823373794555664e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8e-07,
      "loss": -0.073,
      "reward": -0.3014819361269474,
      "reward_after_mean": -0.3014819361269474,
      "reward_after_std": 0.4007901232689619,
      "reward_before_mean": -0.11112237721681595,
      "reward_before_std": 0.36138677038252354,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.1903595570474863,
      "reward_change_min": -0.3259355407208204,
      "reward_change_std": 0.11471330747008324,
      "reward_std": 0.4007901381701231,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/cosine_scaled_reward": -0.17362238001078367,
      "step": 9
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2706.750045776367,
      "epoch": 0.011428571428571429,
      "grad_norm": 0.07514520734548569,
      "kl": 4.935264587402344e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9e-07,
      "loss": 0.0161,
      "reward": -0.11493759602308273,
      "reward_after_mean": -0.11493759602308273,
      "reward_after_std": 0.5900738928467035,
      "reward_before_mean": 0.12419397698249668,
      "reward_before_std": 0.599910007789731,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.23913157917559147,
      "reward_change_min": -0.4677496124058962,
      "reward_change_std": 0.18292754143476486,
      "reward_std": 0.5900739189237356,
      "rewards/accuracy_reward": 0.20833333767950535,
      "rewards/cosine_scaled_reward": -0.08413936011493206,
      "step": 10
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3402.229217529297,
      "epoch": 0.012571428571428572,
      "grad_norm": 0.05007028952240944,
      "kl": 3.884732723236084e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1e-06,
      "loss": 0.0793,
      "reward": -0.29482845464372076,
      "reward_after_mean": -0.29482845464372076,
      "reward_after_std": 0.474648579955101,
      "reward_before_mean": -0.11345059424638748,
      "reward_before_std": 0.4228599341586232,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.18137785978615284,
      "reward_change_min": -0.3095668628811836,
      "reward_change_std": 0.109623190946877,
      "reward_std": 0.47464858181774616,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/cosine_scaled_reward": -0.19678393006324768,
      "step": 11
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2166.041702270508,
      "epoch": 0.013714285714285714,
      "grad_norm": 0.08323562890291214,
      "kl": 5.860626697540283e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.997258721585931e-07,
      "loss": 0.02,
      "reward": -0.11367994034662843,
      "reward_after_mean": -0.11367994034662843,
      "reward_after_std": 0.5573214925825596,
      "reward_before_mean": 0.1285952888429165,
      "reward_before_std": 0.5449656378477812,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.24227523803710938,
      "reward_change_min": -0.45779313147068024,
      "reward_change_std": 0.1707105627283454,
      "reward_std": 0.5573215000331402,
      "rewards/accuracy_reward": 0.18750000558793545,
      "rewards/cosine_scaled_reward": -0.058904721518047154,
      "step": 12
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2882.1041717529297,
      "epoch": 0.014857142857142857,
      "grad_norm": 0.06770065426826477,
      "kl": 4.939734935760498e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.989038226169207e-07,
      "loss": 0.0159,
      "reward": -0.04806898767128587,
      "reward_after_mean": -0.04806898767128587,
      "reward_after_std": 0.41037579998373985,
      "reward_before_mean": 0.23718727007508278,
      "reward_before_std": 0.3681218596175313,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2852562852203846,
      "reward_change_min": -0.4630883075296879,
      "reward_change_std": 0.17784415930509567,
      "reward_std": 0.4103758055716753,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/cosine_scaled_reward": -0.012812718749046326,
      "step": 13
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2489.9583587646484,
      "epoch": 0.016,
      "grad_norm": 0.07574188709259033,
      "kl": 4.4599175453186035e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.975348529157229e-07,
      "loss": -0.0047,
      "reward": -0.27982987742871046,
      "reward_after_mean": -0.27982987742871046,
      "reward_after_std": 0.39828755520284176,
      "reward_before_mean": -0.07930825836956501,
      "reward_before_std": 0.36426794342696667,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.20052163302898407,
      "reward_change_min": -0.3274822384119034,
      "reward_change_std": 0.12421673815697432,
      "reward_std": 0.3982875719666481,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/cosine_scaled_reward": -0.16264159604907036,
      "step": 14
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2756.062557220459,
      "epoch": 0.017142857142857144,
      "grad_norm": 0.07491511106491089,
      "kl": 4.3898820877075195e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.956206309337066e-07,
      "loss": -0.0053,
      "reward": -0.02700677514076233,
      "reward_after_mean": -0.02700677514076233,
      "reward_after_std": 0.4231729060411453,
      "reward_before_mean": 0.26372290030121803,
      "reward_before_std": 0.37921188212931156,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.290729658678174,
      "reward_change_min": -0.4885778911411762,
      "reward_change_std": 0.1829922767356038,
      "reward_std": 0.42317293770611286,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/cosine_scaled_reward": 0.013722889125347137,
      "step": 15
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3446.500030517578,
      "epoch": 0.018285714285714287,
      "grad_norm": 0.05100657790899277,
      "kl": 5.772709846496582e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.931634888554935e-07,
      "loss": 0.0162,
      "reward": -0.2724580317735672,
      "reward_after_mean": -0.2724580317735672,
      "reward_after_std": 0.391301766037941,
      "reward_before_mean": -0.0656847816426307,
      "reward_before_std": 0.3727356269955635,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.20677325874567032,
      "reward_change_min": -0.33162772841751575,
      "reward_change_std": 0.13028155453503132,
      "reward_std": 0.391301779076457,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.16985145024955273,
      "step": 16
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2186.333351135254,
      "epoch": 0.019428571428571427,
      "grad_norm": 0.1111261397600174,
      "kl": 4.7326087951660156e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.901664203302124e-07,
      "loss": -0.0433,
      "reward": -0.09547922573983669,
      "reward_after_mean": -0.09547922573983669,
      "reward_after_std": 0.49696200527250767,
      "reward_before_mean": 0.16047327406704426,
      "reward_before_std": 0.4803733183071017,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.25595249235630035,
      "reward_change_min": -0.4563688188791275,
      "reward_change_std": 0.17162334639579058,
      "reward_std": 0.4969620108604431,
      "rewards/accuracy_reward": 0.20833334140479565,
      "rewards/cosine_scaled_reward": -0.047860062681138515,
      "step": 17
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3001.8959045410156,
      "epoch": 0.02057142857142857,
      "grad_norm": 0.04722139984369278,
      "kl": 3.723800182342529e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.866330768241983e-07,
      "loss": 0.0773,
      "reward": -0.0474961269646883,
      "reward_after_mean": -0.0474961269646883,
      "reward_after_std": 0.6040066350251436,
      "reward_before_mean": 0.20818753214552999,
      "reward_before_std": 0.550414253026247,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.25568367168307304,
      "reward_change_min": -0.4698070529848337,
      "reward_change_std": 0.169005892239511,
      "reward_std": 0.6040066443383694,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/cosine_scaled_reward": -0.04181248042732477,
      "step": 18
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2931.2917098999023,
      "epoch": 0.021714285714285714,
      "grad_norm": 0.06986711174249649,
      "kl": 3.400444984436035e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.0292,
      "reward": 0.17090284021105617,
      "reward_after_mean": 0.17090284021105617,
      "reward_after_std": 0.5971331372857094,
      "reward_before_mean": 0.5100600440055132,
      "reward_before_std": 0.5602419087663293,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.33915719762444496,
      "reward_change_min": -0.5498323440551758,
      "reward_change_std": 0.21470873430371284,
      "reward_std": 0.5971331428736448,
      "rewards/accuracy_reward": 0.35416667349636555,
      "rewards/cosine_scaled_reward": 0.15589335677213967,
      "step": 19
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 1947.3333854675293,
      "epoch": 0.022857142857142857,
      "grad_norm": 0.09310275316238403,
      "kl": 2.9101967811584473e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.047,
      "reward": -0.030604678206145763,
      "reward_after_mean": -0.030604678206145763,
      "reward_after_std": 0.5643759882077575,
      "reward_before_mean": 0.23468416556715965,
      "reward_before_std": 0.5034359507262707,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.265288844704628,
      "reward_change_min": -0.4424171634018421,
      "reward_change_std": 0.16660070698708296,
      "reward_std": 0.5643760003149509,
      "rewards/accuracy_reward": 0.2708333358168602,
      "rewards/cosine_scaled_reward": -0.036149172694422305,
      "step": 20
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2655.0625343322754,
      "epoch": 0.024,
      "grad_norm": 0.10517873615026474,
      "kl": 4.278123378753662e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.728616793536587e-07,
      "loss": -0.0291,
      "reward": -0.10848977044224739,
      "reward_after_mean": -0.10848977044224739,
      "reward_after_std": 0.5077532883733511,
      "reward_before_mean": 0.13500482868403196,
      "reward_before_std": 0.4411742137745023,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.24349460750818253,
      "reward_change_min": -0.3907940573990345,
      "reward_change_std": 0.14410312101244926,
      "reward_std": 0.5077533088624477,
      "rewards/accuracy_reward": 0.2083333358168602,
      "rewards/cosine_scaled_reward": -0.07332850294187665,
      "step": 21
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 1586.75004196167,
      "epoch": 0.025142857142857144,
      "grad_norm": 0.1125827431678772,
      "kl": 3.287568688392639e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.672327345550543e-07,
      "loss": -0.0943,
      "reward": -0.019220076501369476,
      "reward_after_mean": -0.019220076501369476,
      "reward_after_std": 0.38232380524277687,
      "reward_before_mean": 0.2721855500712991,
      "reward_before_std": 0.2772734249010682,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2914056619629264,
      "reward_change_min": -0.43606279231607914,
      "reward_change_std": 0.16009643021970987,
      "reward_std": 0.3823238220065832,
      "rewards/accuracy_reward": 0.2916666679084301,
      "rewards/cosine_scaled_reward": -0.019481111317873,
      "step": 22
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2185.0208702087402,
      "epoch": 0.026285714285714287,
      "grad_norm": 0.08662360906600952,
      "kl": 3.97413969039917e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.610954559391704e-07,
      "loss": 0.0316,
      "reward": 0.0812848350033164,
      "reward_after_mean": 0.0812848350033164,
      "reward_after_std": 0.7008634004741907,
      "reward_before_mean": 0.3680048354435712,
      "reward_before_std": 0.6547262277454138,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.28671999275684357,
      "reward_change_min": -0.5204410944133997,
      "reward_change_std": 0.19234656170010567,
      "reward_std": 0.7008634451776743,
      "rewards/accuracy_reward": 0.3125000037252903,
      "rewards/cosine_scaled_reward": 0.05550480890087783,
      "step": 23
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2429.7500381469727,
      "epoch": 0.027428571428571427,
      "grad_norm": 0.09173740446567535,
      "kl": 3.2924115657806396e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.54457320834625e-07,
      "loss": 0.0745,
      "reward": 0.1347651109099388,
      "reward_after_mean": 0.1347651109099388,
      "reward_after_std": 0.6127859707921743,
      "reward_before_mean": 0.4589935354888439,
      "reward_before_std": 0.583356948569417,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.3242284394800663,
      "reward_change_min": -0.5463455878198147,
      "reward_change_std": 0.21345821302384138,
      "reward_std": 0.6127859856933355,
      "rewards/accuracy_reward": 0.35416667349636555,
      "rewards/cosine_scaled_reward": 0.10482687130570412,
      "step": 24
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2295.8125228881836,
      "epoch": 0.02857142857142857,
      "grad_norm": 0.08578181266784668,
      "kl": 5.142390727996826e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.473264167865171e-07,
      "loss": -0.0378,
      "reward": -0.24743555346503854,
      "reward_after_mean": -0.24743555346503854,
      "reward_after_std": 0.47581059113144875,
      "reward_before_mean": -0.044429176254197955,
      "reward_before_std": 0.4596579996868968,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.20300637558102608,
      "reward_change_min": -0.3940556962043047,
      "reward_change_std": 0.13973176665604115,
      "reward_std": 0.47581060975790024,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.14859584486111999,
      "step": 25
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2757.541748046875,
      "epoch": 0.029714285714285714,
      "grad_norm": 0.062217455357313156,
      "kl": 3.5900622606277466e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.397114317029974e-07,
      "loss": -0.0243,
      "reward": -0.22815486788749695,
      "reward_after_mean": -0.22815486788749695,
      "reward_after_std": 0.23574577271938324,
      "reward_before_mean": 0.010172484442591667,
      "reward_before_std": 0.1596730425953865,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.23832733370363712,
      "reward_change_min": -0.3241576459258795,
      "reward_change_std": 0.12215171847492456,
      "reward_std": 0.23574578203260899,
      "rewards/accuracy_reward": 0.125,
      "rewards/cosine_scaled_reward": -0.11482751555740833,
      "step": 26
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2858.062545776367,
      "epoch": 0.030857142857142857,
      "grad_norm": 0.07970089465379715,
      "kl": 6.517767906188965e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.0278,
      "reward": -0.22820639377459884,
      "reward_after_mean": -0.22820639377459884,
      "reward_after_std": 0.3691936992108822,
      "reward_before_mean": -0.003585641272366047,
      "reward_before_std": 0.3326329058036208,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.22462073341012,
      "reward_change_min": -0.3414224237203598,
      "reward_change_std": 0.1343115232884884,
      "reward_std": 0.36919371597468853,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.10775232722517103,
      "step": 27
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2576.958366394043,
      "epoch": 0.032,
      "grad_norm": 0.08655045926570892,
      "kl": 5.263090133666992e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.230669076497687e-07,
      "loss": 0.0032,
      "reward": -0.1358381314203143,
      "reward_after_mean": -0.1358381314203143,
      "reward_after_std": 0.5688246618956327,
      "reward_before_mean": 0.0954693965613842,
      "reward_before_std": 0.5578853543847799,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.23130753822624683,
      "reward_change_min": -0.4282049834728241,
      "reward_change_std": 0.16099752951413393,
      "reward_std": 0.5688246786594391,
      "rewards/accuracy_reward": 0.20833333767950535,
      "rewards/cosine_scaled_reward": -0.11286392994225025,
      "step": 28
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2898.750030517578,
      "epoch": 0.03314285714285714,
      "grad_norm": 0.06516330689191818,
      "kl": 4.172325134277344e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.140576474687263e-07,
      "loss": -0.0722,
      "reward": -0.3742859149351716,
      "reward_after_mean": -0.3742859149351716,
      "reward_after_std": 0.32394181191921234,
      "reward_before_mean": -0.19833002239465714,
      "reward_before_std": 0.295702856965363,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.1759559065103531,
      "reward_change_min": -0.31070076301693916,
      "reward_change_std": 0.1110200947150588,
      "reward_std": 0.32394181936979294,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/cosine_scaled_reward": -0.23999668890610337,
      "step": 29
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2727.916763305664,
      "epoch": 0.03428571428571429,
      "grad_norm": 0.08452288061380386,
      "kl": 4.813075065612793e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 9.046048391230247e-07,
      "loss": 0.0229,
      "reward": 0.0020619072020053864,
      "reward_after_mean": 0.0020619072020053864,
      "reward_after_std": 0.6899437569081783,
      "reward_before_mean": 0.27052484080195427,
      "reward_before_std": 0.7042382340878248,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2684629485011101,
      "reward_change_min": -0.5466452892869711,
      "reward_change_std": 0.20367024186998606,
      "reward_std": 0.6899437606334686,
      "rewards/accuracy_reward": 0.25000000558793545,
      "rewards/cosine_scaled_reward": 0.02052484266459942,
      "step": 30
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3069.0208435058594,
      "epoch": 0.03542857142857143,
      "grad_norm": 0.06624361127614975,
      "kl": 5.201995372772217e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.0393,
      "reward": -0.18817763216793537,
      "reward_after_mean": -0.18817763216793537,
      "reward_after_std": 0.4799848888069391,
      "reward_before_mean": 0.03944748570211232,
      "reward_before_std": 0.4816620806232095,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.22762510925531387,
      "reward_change_min": -0.39297138154506683,
      "reward_change_std": 0.16033693589270115,
      "reward_std": 0.4799849148839712,
      "rewards/accuracy_reward": 0.16666666977107525,
      "rewards/cosine_scaled_reward": -0.12721917685121298,
      "step": 31
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2598.854248046875,
      "epoch": 0.036571428571428574,
      "grad_norm": 0.06277475506067276,
      "kl": 4.087388515472412e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.844151714648274e-07,
      "loss": -0.013,
      "reward": -0.026365877129137516,
      "reward_after_mean": -0.026365877129137516,
      "reward_after_std": 0.5388314640149474,
      "reward_before_mean": 0.2528522349894047,
      "reward_before_std": 0.5446693664416671,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.27921812795102596,
      "reward_change_min": -0.5132879670709372,
      "reward_change_std": 0.1965640289708972,
      "reward_std": 0.5388314817100763,
      "rewards/accuracy_reward": 0.2708333432674408,
      "rewards/cosine_scaled_reward": -0.017981095821596682,
      "step": 32
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3199.604202270508,
      "epoch": 0.037714285714285714,
      "grad_norm": 0.05635827034711838,
      "kl": 4.775822162628174e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.737029101523929e-07,
      "loss": 0.0363,
      "reward": -0.11553127923980355,
      "reward_after_mean": -0.11553127923980355,
      "reward_after_std": 0.6909586228430271,
      "reward_before_mean": 0.10691910376772285,
      "reward_before_std": 0.6950575169175863,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.22245037741959095,
      "reward_change_min": -0.4544982947409153,
      "reward_change_std": 0.1732303500175476,
      "reward_std": 0.690958658233285,
      "rewards/accuracy_reward": 0.2083333358168602,
      "rewards/cosine_scaled_reward": -0.10141423298045993,
      "step": 33
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2291.1875076293945,
      "epoch": 0.038857142857142854,
      "grad_norm": 0.07628920674324036,
      "kl": 5.2928924560546875e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.625962667065487e-07,
      "loss": -0.0126,
      "reward": 0.11718782410025597,
      "reward_after_mean": 0.11718782410025597,
      "reward_after_std": 0.6173261571675539,
      "reward_before_mean": 0.4387818221002817,
      "reward_before_std": 0.6273570032790303,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.3215939924120903,
      "reward_change_min": -0.6037725955247879,
      "reward_change_std": 0.226388999260962,
      "reward_std": 0.6173261664807796,
      "rewards/accuracy_reward": 0.3333333469927311,
      "rewards/cosine_scaled_reward": 0.10544847697019577,
      "step": 34
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2999.6666870117188,
      "epoch": 0.04,
      "grad_norm": 0.09039846807718277,
      "kl": 6.504356861114502e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.511087728614862e-07,
      "loss": -0.0028,
      "reward": -0.16610824968665838,
      "reward_after_mean": -0.16610824968665838,
      "reward_after_std": 0.58190375007689,
      "reward_before_mean": 0.05111471749842167,
      "reward_before_std": 0.5643127737566829,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.21722296811640263,
      "reward_change_min": -0.40635400637984276,
      "reward_change_std": 0.15069269575178623,
      "reward_std": 0.5819037612527609,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/cosine_scaled_reward": -0.09471861086785793,
      "step": 35
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3241.6041870117188,
      "epoch": 0.04114285714285714,
      "grad_norm": 0.05941932648420334,
      "kl": 4.7147274017333984e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.392544243589427e-07,
      "loss": 0.0446,
      "reward": -0.3385976613499224,
      "reward_after_mean": -0.3385976613499224,
      "reward_after_std": 0.4202655293047428,
      "reward_before_mean": -0.16420376114547253,
      "reward_before_std": 0.3881509117782116,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.17439390905201435,
      "reward_change_min": -0.31771647930145264,
      "reward_change_std": 0.11316250916570425,
      "reward_std": 0.42026553489267826,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/cosine_scaled_reward": -0.22670375928282738,
      "step": 36
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3407.250030517578,
      "epoch": 0.04228571428571429,
      "grad_norm": 0.05422111600637436,
      "kl": 5.02467155456543e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.270476638965461e-07,
      "loss": 0.0107,
      "reward": -0.3340600021183491,
      "reward_after_mean": -0.3340600021183491,
      "reward_after_std": 0.27687052451074123,
      "reward_before_mean": -0.1344250589609146,
      "reward_before_std": 0.24589443486183882,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.19963493384420872,
      "reward_change_min": -0.31763908080756664,
      "reward_change_std": 0.11714739445596933,
      "reward_std": 0.2768705263733864,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/cosine_scaled_reward": -0.2177584059536457,
      "step": 37
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3299.625,
      "epoch": 0.04342857142857143,
      "grad_norm": 0.058690257370471954,
      "kl": 4.3392181396484375e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.145033635316128e-07,
      "loss": -0.0289,
      "reward": -0.1949592368910089,
      "reward_after_mean": -0.1949592368910089,
      "reward_after_std": 0.3663552775979042,
      "reward_before_mean": 0.040207072626799345,
      "reward_before_std": 0.32383421063423157,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2351663038134575,
      "reward_change_min": -0.3440061006695032,
      "reward_change_std": 0.13658129330724478,
      "reward_std": 0.36635528318583965,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.06395959993824363,
      "step": 38
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2834.208354949951,
      "epoch": 0.044571428571428574,
      "grad_norm": 0.06519138813018799,
      "kl": 5.877390503883362e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 8.01636806561836e-07,
      "loss": 0.0434,
      "reward": 0.0075050946325063705,
      "reward_after_mean": 0.0075050946325063705,
      "reward_after_std": 0.3204457741230726,
      "reward_before_mean": 0.31809414783492684,
      "reward_before_std": 0.20561318658292294,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.3105890806764364,
      "reward_change_min": -0.4078735690563917,
      "reward_change_std": 0.157848265953362,
      "reward_std": 0.3204457778483629,
      "rewards/accuracy_reward": 0.27083333395421505,
      "rewards/cosine_scaled_reward": 0.047260792925953865,
      "step": 39
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2497.2708892822266,
      "epoch": 0.045714285714285714,
      "grad_norm": 0.06621085107326508,
      "kl": 3.94284725189209e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7.884636689049422e-07,
      "loss": 0.0453,
      "reward": -0.09613560698926449,
      "reward_after_mean": -0.09613560698926449,
      "reward_after_std": 0.47081254981458187,
      "reward_before_mean": 0.15380273573100567,
      "reward_before_std": 0.37723068334162235,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.249938340857625,
      "reward_change_min": -0.34136574156582355,
      "reward_change_std": 0.13006254099309444,
      "reward_std": 0.4708125591278076,
      "rewards/accuracy_reward": 0.18750000186264515,
      "rewards/cosine_scaled_reward": -0.0336972763761878,
      "step": 40
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2959.541702270508,
      "epoch": 0.046857142857142854,
      "grad_norm": 0.05511856451630592,
      "kl": 4.7087669372558594e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7.75e-07,
      "loss": 0.0083,
      "reward": -0.18304359819740057,
      "reward_after_mean": -0.18304359819740057,
      "reward_after_std": 0.5882194386795163,
      "reward_before_mean": 0.027974394150078297,
      "reward_before_std": 0.5736249866895378,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.21101799607276917,
      "reward_change_min": -0.40425144881010056,
      "reward_change_std": 0.1514134258031845,
      "reward_std": 0.588219441473484,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/cosine_scaled_reward": -0.11785894399508834,
      "step": 41
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2838.7916774749756,
      "epoch": 0.048,
      "grad_norm": 0.10495254397392273,
      "kl": 6.80088996887207e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7.612622032536507e-07,
      "loss": 0.0644,
      "reward": -0.4437730088829994,
      "reward_after_mean": -0.4437730088829994,
      "reward_after_std": 0.1842181822285056,
      "reward_before_mean": -0.2758090700954199,
      "reward_before_std": 0.14354530815035105,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.16796394996345043,
      "reward_change_min": -0.23861237615346909,
      "reward_change_std": 0.0892399987205863,
      "reward_std": 0.1842181896790862,
      "rewards/accuracy_reward": 0.0,
      "rewards/cosine_scaled_reward": -0.2758090700954199,
      "step": 42
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3059.2916870117188,
      "epoch": 0.04914285714285714,
      "grad_norm": 0.06792694330215454,
      "kl": 4.3764710426330566e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7.472670160550848e-07,
      "loss": -0.0857,
      "reward": -0.3225740883499384,
      "reward_after_mean": -0.3225740883499384,
      "reward_after_std": 0.37870580051094294,
      "reward_before_mean": -0.1348972450941801,
      "reward_before_std": 0.3481356706470251,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.18767685443162918,
      "reward_change_min": -0.3287505563348532,
      "reward_change_std": 0.11681838473305106,
      "reward_std": 0.37870581168681383,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/cosine_scaled_reward": -0.19739723904058337,
      "step": 43
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2378.020851135254,
      "epoch": 0.05028571428571429,
      "grad_norm": 0.0813366174697876,
      "kl": 3.6869198083877563e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7.330314893841101e-07,
      "loss": 0.0455,
      "reward": -0.05170046165585518,
      "reward_after_mean": -0.05170046165585518,
      "reward_after_std": 0.4305565822869539,
      "reward_before_mean": 0.22961101355031133,
      "reward_before_std": 0.41200026869773865,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.28131146915256977,
      "reward_change_min": -0.4182246755808592,
      "reward_change_std": 0.16517735086381435,
      "reward_std": 0.43055659532546997,
      "rewards/accuracy_reward": 0.22916667722165585,
      "rewards/cosine_scaled_reward": 0.0004443284124135971,
      "step": 44
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3216.812530517578,
      "epoch": 0.05142857142857143,
      "grad_norm": 0.05646404251456261,
      "kl": 4.851818084716797e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7.185729670371604e-07,
      "loss": -0.0235,
      "reward": -0.2375551089644432,
      "reward_after_mean": -0.2375551089644432,
      "reward_after_std": 0.42938368022441864,
      "reward_before_mean": -0.027880379930138588,
      "reward_before_std": 0.39119122084230185,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.20967471785843372,
      "reward_change_min": -0.30014154128730297,
      "reward_change_std": 0.11865215376019478,
      "reward_std": 0.42938369885087013,
      "rewards/accuracy_reward": 0.1458333395421505,
      "rewards/cosine_scaled_reward": -0.1737137222662568,
      "step": 45
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3094.2500610351562,
      "epoch": 0.052571428571428575,
      "grad_norm": 0.06780198961496353,
      "kl": 5.8978796005249023e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 7.039090644965509e-07,
      "loss": 0.0149,
      "reward": -0.3860484268516302,
      "reward_after_mean": -0.3860484268516302,
      "reward_after_std": 0.2960197441279888,
      "reward_before_mean": -0.21315561048686504,
      "reward_before_std": 0.24969620257616043,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.17289282009005547,
      "reward_change_min": -0.2534465938806534,
      "reward_change_std": 0.09384610410779715,
      "reward_std": 0.29601974971592426,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/cosine_scaled_reward": -0.2339889481663704,
      "step": 46
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2600.062515258789,
      "epoch": 0.053714285714285714,
      "grad_norm": 0.09613389521837234,
      "kl": 4.597008228302002e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 6.890576474687263e-07,
      "loss": -0.0031,
      "reward": -0.15789308166131377,
      "reward_after_mean": -0.15789308166131377,
      "reward_after_std": 0.4657074324786663,
      "reward_before_mean": 0.07849103212356567,
      "reward_before_std": 0.44048627745360136,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2363841012120247,
      "reward_change_min": -0.4060584995895624,
      "reward_change_std": 0.15053978469222784,
      "reward_std": 0.46570744924247265,
      "rewards/accuracy_reward": 0.16666667349636555,
      "rewards/cosine_scaled_reward": -0.0881756441667676,
      "step": 47
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2541.0208892822266,
      "epoch": 0.054857142857142854,
      "grad_norm": 0.08367791026830673,
      "kl": 4.8547983169555664e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 6.740368101176495e-07,
      "loss": -0.0132,
      "reward": -0.04096842650324106,
      "reward_after_mean": -0.04096842650324106,
      "reward_after_std": 0.6091550681740046,
      "reward_before_mean": 0.22568438947200775,
      "reward_before_std": 0.6412058565765619,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.26665280386805534,
      "reward_change_min": -0.540655393153429,
      "reward_change_std": 0.2099922765046358,
      "reward_std": 0.6091550793498755,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/cosine_scaled_reward": -0.024315630551427603,
      "step": 48
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 1905.8333625793457,
      "epoch": 0.056,
      "grad_norm": 0.10268070548772812,
      "kl": 3.053247928619385e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 6.588648530198504e-07,
      "loss": -0.0009,
      "reward": -0.15249623358249664,
      "reward_after_mean": -0.15249623358249664,
      "reward_after_std": 0.326967298053205,
      "reward_before_mean": 0.1072658714838326,
      "reward_before_std": 0.3044881981331855,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2597620990127325,
      "reward_change_min": -0.3807704448699951,
      "reward_change_std": 0.15653882548213005,
      "reward_std": 0.32696729991585016,
      "rewards/accuracy_reward": 0.1875000074505806,
      "rewards/cosine_scaled_reward": -0.08023415133357048,
      "step": 49
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3011.4166984558105,
      "epoch": 0.05714285714285714,
      "grad_norm": 0.07368293404579163,
      "kl": 3.7729740142822266e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 6.435602608679916e-07,
      "loss": 0.0181,
      "reward": -0.030200980603694916,
      "reward_after_mean": -0.030200980603694916,
      "reward_after_std": 0.4144356604665518,
      "reward_before_mean": 0.26172044314444065,
      "reward_before_std": 0.3721779463812709,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.291921429336071,
      "reward_change_min": -0.4539058431982994,
      "reward_change_std": 0.18305509351193905,
      "reward_std": 0.4144356790930033,
      "rewards/accuracy_reward": 0.2708333358168602,
      "rewards/cosine_scaled_reward": -0.00911291316151619,
      "step": 50
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2326.833351135254,
      "epoch": 0.05828571428571429,
      "grad_norm": 0.09606306254863739,
      "kl": 4.869699478149414e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 6.281416799501187e-07,
      "loss": -0.0003,
      "reward": -0.1915714619681239,
      "reward_after_mean": -0.1915714619681239,
      "reward_after_std": 0.4830855615437031,
      "reward_before_mean": 0.03159985225647688,
      "reward_before_std": 0.4657090399414301,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.22317130863666534,
      "reward_change_min": -0.38562823459506035,
      "reward_change_std": 0.1446484811604023,
      "reward_std": 0.4830855708569288,
      "rewards/accuracy_reward": 0.14583333767950535,
      "rewards/cosine_scaled_reward": -0.1142334844917059,
      "step": 51
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2670.354217529297,
      "epoch": 0.05942857142857143,
      "grad_norm": 0.09154725819826126,
      "kl": 3.1536445021629333e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 6.126278954320294e-07,
      "loss": 0.0598,
      "reward": 0.19081778265535831,
      "reward_after_mean": 0.19081778265535831,
      "reward_after_std": 0.6043459214270115,
      "reward_before_mean": 0.5438746921718121,
      "reward_before_std": 0.6257130298763514,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.3530569151043892,
      "reward_change_min": -0.5976719129830599,
      "reward_change_std": 0.24177804868668318,
      "reward_std": 0.6043459549546242,
      "rewards/accuracy_reward": 0.4166666828095913,
      "rewards/cosine_scaled_reward": 0.12720800563693047,
      "step": 52
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2716.104202270508,
      "epoch": 0.060571428571428575,
      "grad_norm": 0.08996250480413437,
      "kl": 4.64320182800293e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5.97037808470444e-07,
      "loss": 0.0028,
      "reward": -0.003628397360444069,
      "reward_after_mean": -0.003628397360444069,
      "reward_after_std": 0.6251984257251024,
      "reward_before_mean": 0.2679465925320983,
      "reward_before_std": 0.6042734682559967,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.271575003862381,
      "reward_change_min": -0.42408623918890953,
      "reward_change_std": 0.1748236557468772,
      "reward_std": 0.625198433175683,
      "rewards/accuracy_reward": 0.22916667349636555,
      "rewards/cosine_scaled_reward": 0.038779920781962574,
      "step": 53
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2338.2500534057617,
      "epoch": 0.061714285714285715,
      "grad_norm": 0.09756185859441757,
      "kl": 4.079937934875488e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5.813904131848564e-07,
      "loss": 0.081,
      "reward": 0.20069499779492617,
      "reward_after_mean": 0.20069499779492617,
      "reward_after_std": 0.7350467648357153,
      "reward_before_mean": 0.5314708258956671,
      "reward_before_std": 0.7071247743442655,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.3307758532464504,
      "reward_change_min": -0.5639360137283802,
      "reward_change_std": 0.22586506605148315,
      "reward_std": 0.7350467666983604,
      "rewards/accuracy_reward": 0.3750000037252903,
      "rewards/cosine_scaled_reward": 0.15647082310169935,
      "step": 54
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2827.208381652832,
      "epoch": 0.06285714285714286,
      "grad_norm": 0.06813458353281021,
      "kl": 3.8139522075653076e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5.657047735161255e-07,
      "loss": 0.0192,
      "reward": 0.01979338377714157,
      "reward_after_mean": 0.01979338377714157,
      "reward_after_std": 0.47581884637475014,
      "reward_before_mean": 0.31956043280661106,
      "reward_before_std": 0.433451721444726,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2997670602053404,
      "reward_change_min": -0.4422443322837353,
      "reward_change_std": 0.17958854231983423,
      "reward_std": 0.4758188594132662,
      "rewards/accuracy_reward": 0.31250000558793545,
      "rewards/cosine_scaled_reward": 0.007060416042804718,
      "step": 55
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3146.187545776367,
      "epoch": 0.064,
      "grad_norm": 0.07230962067842484,
      "kl": 4.698336124420166e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5.5e-07,
      "loss": 0.0041,
      "reward": -0.3684567688032985,
      "reward_after_mean": -0.3684567688032985,
      "reward_after_std": 0.3458494506776333,
      "reward_before_mean": -0.19647526927292347,
      "reward_before_std": 0.2858909470960498,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.17198150791227818,
      "reward_change_min": -0.25032067485153675,
      "reward_change_std": 0.09047863632440567,
      "reward_std": 0.3458494581282139,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/cosine_scaled_reward": -0.23814193485304713,
      "step": 56
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3092.4583892822266,
      "epoch": 0.06514285714285714,
      "grad_norm": 0.054168950766325,
      "kl": 3.5762786865234375e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5.342952264838747e-07,
      "loss": -0.0577,
      "reward": -0.22174427472054958,
      "reward_after_mean": -0.22174427472054958,
      "reward_after_std": 0.4740599449723959,
      "reward_before_mean": -0.0071051958948373795,
      "reward_before_std": 0.46773200016468763,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.21463906578719616,
      "reward_change_min": -0.43531039729714394,
      "reward_change_std": 0.1588157471269369,
      "reward_std": 0.47405995428562164,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/cosine_scaled_reward": -0.1529385419562459,
      "step": 57
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2167.9792098999023,
      "epoch": 0.06628571428571428,
      "grad_norm": 0.08869833499193192,
      "kl": 3.4928321838378906e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5.186095868151436e-07,
      "loss": 0.0885,
      "reward": -0.060710884630680084,
      "reward_after_mean": -0.060710884630680084,
      "reward_after_std": 0.35876573994755745,
      "reward_before_mean": 0.21950972266495228,
      "reward_before_std": 0.28317089146003127,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.28022063709795475,
      "reward_change_min": -0.3964573312550783,
      "reward_change_std": 0.14705425314605236,
      "reward_std": 0.3587657529860735,
      "rewards/accuracy_reward": 0.27083333395421505,
      "rewards/cosine_scaled_reward": -0.051323600113391876,
      "step": 58
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2832.645851135254,
      "epoch": 0.06742857142857143,
      "grad_norm": 0.06335015594959259,
      "kl": 3.361701965332031e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 5.02962191529556e-07,
      "loss": -0.0151,
      "reward": -0.3008502349257469,
      "reward_after_mean": -0.3008502349257469,
      "reward_after_std": 0.46144136413931847,
      "reward_before_mean": -0.11781154479831457,
      "reward_before_std": 0.4245893997140229,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.18303867429494858,
      "reward_change_min": -0.32882457226514816,
      "reward_change_std": 0.11585518904030323,
      "reward_std": 0.46144137158989906,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/cosine_scaled_reward": -0.20114488480612636,
      "step": 59
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2861.604217529297,
      "epoch": 0.06857142857142857,
      "grad_norm": 0.06764096766710281,
      "kl": 4.756450653076172e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 4.873721045679706e-07,
      "loss": 0.0064,
      "reward": -0.3513468327000737,
      "reward_after_mean": -0.3513468327000737,
      "reward_after_std": 0.34203874319791794,
      "reward_before_mean": -0.17035986855626106,
      "reward_before_std": 0.3069671168923378,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.18098697252571583,
      "reward_change_min": -0.3226324897259474,
      "reward_change_std": 0.11291467119008303,
      "reward_std": 0.34203874692320824,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/cosine_scaled_reward": -0.23285987111739814,
      "step": 60
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3003.4166870117188,
      "epoch": 0.06971428571428571,
      "grad_norm": 0.05480307340621948,
      "kl": 3.312528133392334e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.0558,
      "reward": 0.07884835358709097,
      "reward_after_mean": 0.07884835358709097,
      "reward_after_std": 0.5404287055134773,
      "reward_before_mean": 0.39688345044851303,
      "reward_before_std": 0.5404559625312686,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.31803508289158344,
      "reward_change_min": -0.5349768288433552,
      "reward_change_std": 0.21121840458363295,
      "reward_std": 0.5404287241399288,
      "rewards/accuracy_reward": 0.3125000111758709,
      "rewards/cosine_scaled_reward": 0.0843834443949163,
      "step": 61
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2324.333381652832,
      "epoch": 0.07085714285714285,
      "grad_norm": 0.08491574972867966,
      "kl": 3.9268285036087036e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 4.5643973913200837e-07,
      "loss": -0.0186,
      "reward": 0.05790156498551369,
      "reward_after_mean": 0.05790156498551369,
      "reward_after_std": 0.5291469320654869,
      "reward_before_mean": 0.36868745367974043,
      "reward_before_std": 0.5278607029467821,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.3107858970761299,
      "reward_change_min": -0.4988170899450779,
      "reward_change_std": 0.19942885264754295,
      "reward_std": 0.5291469544172287,
      "rewards/accuracy_reward": 0.31250000558793545,
      "rewards/cosine_scaled_reward": 0.05618744622915983,
      "step": 62
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 1919.5208892822266,
      "epoch": 0.072,
      "grad_norm": 0.08071651309728622,
      "kl": 4.035234451293945e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 4.4113514698014953e-07,
      "loss": 0.0627,
      "reward": 0.03887452092021704,
      "reward_after_mean": 0.03887452092021704,
      "reward_after_std": 0.6215568911284208,
      "reward_before_mean": 0.3234526915475726,
      "reward_before_std": 0.5673101404681802,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.28457815758883953,
      "reward_change_min": -0.46947887167334557,
      "reward_change_std": 0.18151281960308552,
      "reward_std": 0.6215569227933884,
      "rewards/accuracy_reward": 0.2916666716337204,
      "rewards/cosine_scaled_reward": 0.03178601246327162,
      "step": 63
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2727.958396911621,
      "epoch": 0.07314285714285715,
      "grad_norm": 0.07106328755617142,
      "kl": 4.9501657485961914e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 4.2596318988235037e-07,
      "loss": -0.0233,
      "reward": -0.08374386164359748,
      "reward_after_mean": -0.08374386164359748,
      "reward_after_std": 0.5860113948583603,
      "reward_before_mean": 0.16729693859815598,
      "reward_before_std": 0.5889831483364105,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2510407902300358,
      "reward_change_min": -0.4641798008233309,
      "reward_change_std": 0.18448879569768906,
      "reward_std": 0.5860113985836506,
      "rewards/accuracy_reward": 0.20833333395421505,
      "rewards/cosine_scaled_reward": -0.04103639395907521,
      "step": 64
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2691.208354949951,
      "epoch": 0.07428571428571429,
      "grad_norm": 0.0748182013630867,
      "kl": 4.149973392486572e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.0242,
      "reward": -0.19793469086289406,
      "reward_after_mean": -0.19793469086289406,
      "reward_after_std": 0.34430792182683945,
      "reward_before_mean": 0.03801066428422928,
      "reward_before_std": 0.27941721118986607,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2359453458338976,
      "reward_change_min": -0.37367955408990383,
      "reward_change_std": 0.13625783286988735,
      "reward_std": 0.34430794045329094,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/cosine_scaled_reward": -0.12865600176155567,
      "step": 65
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2040.8541736602783,
      "epoch": 0.07542857142857143,
      "grad_norm": 0.12233862280845642,
      "kl": 3.054831176996231e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3.9609093550344907e-07,
      "loss": -0.036,
      "reward": -0.05932062119245529,
      "reward_after_mean": -0.05932062119245529,
      "reward_after_std": 0.37665878515690565,
      "reward_before_mean": 0.22568307630717754,
      "reward_before_std": 0.3319067317061126,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2850037161260843,
      "reward_change_min": -0.4456963036209345,
      "reward_change_std": 0.17067184578627348,
      "reward_std": 0.37665879912674427,
      "rewards/accuracy_reward": 0.2708333395421505,
      "rewards/cosine_scaled_reward": -0.04515025019645691,
      "step": 66
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3306.7083435058594,
      "epoch": 0.07657142857142857,
      "grad_norm": 0.053935784846544266,
      "kl": 4.680454730987549e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3.8142703296283953e-07,
      "loss": -0.0177,
      "reward": -0.47077513113617897,
      "reward_after_mean": -0.47077513113617897,
      "reward_after_std": 0.199712373316288,
      "reward_before_mean": -0.31407430768013,
      "reward_before_std": 0.15584436804056168,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.15670081414282322,
      "reward_change_min": -0.2310307528823614,
      "reward_change_std": 0.08393252175301313,
      "reward_std": 0.19971238262951374,
      "rewards/accuracy_reward": 0.0,
      "rewards/cosine_scaled_reward": -0.31407430768013,
      "step": 67
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 1717.6666870117188,
      "epoch": 0.07771428571428571,
      "grad_norm": 0.1108105406165123,
      "kl": 2.9583927243947983e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3.6696851061588994e-07,
      "loss": -0.0421,
      "reward": -0.23304753471165895,
      "reward_after_mean": -0.23304753471165895,
      "reward_after_std": 0.5989381801337004,
      "reward_before_mean": -0.045263697393238544,
      "reward_before_std": 0.5719784637913108,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.1877838410437107,
      "reward_change_min": -0.38191304728388786,
      "reward_change_std": 0.13277372065931559,
      "reward_std": 0.5989382117986679,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/cosine_scaled_reward": -0.17026370204985142,
      "step": 68
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 1968.9583740234375,
      "epoch": 0.07885714285714286,
      "grad_norm": 0.0938061773777008,
      "kl": 4.889070987701416e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3.5273298394491515e-07,
      "loss": 0.072,
      "reward": -0.29863712191581726,
      "reward_after_mean": -0.29863712191581726,
      "reward_after_std": 0.3859822079539299,
      "reward_before_mean": -0.10575509630143642,
      "reward_before_std": 0.3468486526980996,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.1928820125758648,
      "reward_change_min": -0.3261357322335243,
      "reward_change_std": 0.11793852038681507,
      "reward_std": 0.38598222099244595,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/cosine_scaled_reward": -0.16825510375201702,
      "step": 69
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2932.958366394043,
      "epoch": 0.08,
      "grad_norm": 0.06500542908906937,
      "kl": 4.222989082336426e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3.387377967463493e-07,
      "loss": 0.009,
      "reward": -0.2667752802371979,
      "reward_after_mean": -0.2667752802371979,
      "reward_after_std": 0.30794927291572094,
      "reward_before_mean": -0.04546757601201534,
      "reward_before_std": 0.29563159868121147,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.22130770608782768,
      "reward_change_min": -0.3524601850658655,
      "reward_change_std": 0.13822024781256914,
      "reward_std": 0.30794927291572094,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/cosine_scaled_reward": -0.12880091182887554,
      "step": 70
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2713.7708587646484,
      "epoch": 0.08114285714285714,
      "grad_norm": 0.07305732369422913,
      "kl": 4.410743713378906e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3.250000000000001e-07,
      "loss": -0.0409,
      "reward": -0.16716782632283866,
      "reward_after_mean": -0.16716782632283866,
      "reward_after_std": 0.4499338325113058,
      "reward_before_mean": 0.0670731533318758,
      "reward_before_std": 0.40882042655721307,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2342409696429968,
      "reward_change_min": -0.3438849691301584,
      "reward_change_std": 0.1364891054108739,
      "reward_std": 0.4499338483437896,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/cosine_scaled_reward": -0.05792684169136919,
      "step": 71
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2673.6875762939453,
      "epoch": 0.08228571428571428,
      "grad_norm": 0.11232837289571762,
      "kl": 5.570054054260254e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 3.115363310950578e-07,
      "loss": 0.0121,
      "reward": -0.293288990855217,
      "reward_after_mean": -0.293288990855217,
      "reward_after_std": 0.3026854023337364,
      "reward_before_mean": -0.08422034978866577,
      "reward_before_std": 0.2725341413170099,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2090686447918415,
      "reward_change_min": -0.3304400313645601,
      "reward_change_std": 0.12835939321666956,
      "reward_std": 0.3026854023337364,
      "rewards/accuracy_reward": 0.0625,
      "rewards/cosine_scaled_reward": -0.14672034978866577,
      "step": 72
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3406.3333740234375,
      "epoch": 0.08342857142857144,
      "grad_norm": 0.05645836889743805,
      "kl": 4.6640634536743164e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.9836319343816397e-07,
      "loss": 0.0034,
      "reward": -0.21504229493439198,
      "reward_after_mean": -0.21504229493439198,
      "reward_after_std": 0.2924734726548195,
      "reward_before_mean": 0.02397450990974903,
      "reward_before_std": 0.2590841380879283,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2390167973935604,
      "reward_change_min": -0.36735046096146107,
      "reward_change_std": 0.13649490475654602,
      "reward_std": 0.29247347451746464,
      "rewards/accuracy_reward": 0.1458333395421505,
      "rewards/cosine_scaled_reward": -0.12185884267091751,
      "step": 73
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2586.5416870117188,
      "epoch": 0.08457142857142858,
      "grad_norm": 0.08274800330400467,
      "kl": 5.094707012176514e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.854966364683872e-07,
      "loss": 0.1411,
      "reward": -0.00769616337493062,
      "reward_after_mean": -0.00769616337493062,
      "reward_after_std": 0.5274815298616886,
      "reward_before_mean": 0.27686647698283195,
      "reward_before_std": 0.4944295873865485,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.284562636166811,
      "reward_change_min": -0.5354499518871307,
      "reward_change_std": 0.19903385266661644,
      "reward_std": 0.5274815503507853,
      "rewards/accuracy_reward": 0.2708333358168602,
      "rewards/cosine_scaled_reward": 0.006033138604834676,
      "step": 74
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2864.875015258789,
      "epoch": 0.08571428571428572,
      "grad_norm": 0.06136927381157875,
      "kl": 4.2051076889038086e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.729523361034538e-07,
      "loss": -0.025,
      "reward": 0.0016691870987415314,
      "reward_after_mean": 0.0016691870987415314,
      "reward_after_std": 0.41493271477520466,
      "reward_before_mean": 0.30496746860444546,
      "reward_before_std": 0.36841379571706057,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.30329825915396214,
      "reward_change_min": -0.4862241819500923,
      "reward_change_std": 0.1891703074797988,
      "reward_std": 0.4149327166378498,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/cosine_scaled_reward": 0.05496744904667139,
      "step": 75
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2622.187526702881,
      "epoch": 0.08685714285714285,
      "grad_norm": 0.07091425359249115,
      "kl": 3.7416815757751465e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.6074557564105724e-07,
      "loss": -0.0643,
      "reward": -0.321389552205801,
      "reward_after_mean": -0.321389552205801,
      "reward_after_std": 0.3121167942881584,
      "reward_before_mean": -0.12414262071251869,
      "reward_before_std": 0.2767808083444834,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.19724693521857262,
      "reward_change_min": -0.3346843905746937,
      "reward_change_std": 0.12084136344492435,
      "reward_std": 0.31211681105196476,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/cosine_scaled_reward": -0.1658092886209488,
      "step": 76
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2890.7083587646484,
      "epoch": 0.088,
      "grad_norm": 0.05628199875354767,
      "kl": 4.3079257011413574e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.488912271385139e-07,
      "loss": -0.032,
      "reward": -0.28795215487480164,
      "reward_after_mean": -0.28795215487480164,
      "reward_after_std": 0.30752563290297985,
      "reward_before_mean": -0.07627008855342865,
      "reward_before_std": 0.29114991053938866,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2116820625960827,
      "reward_change_min": -0.34908806160092354,
      "reward_change_std": 0.13322968687862158,
      "reward_std": 0.30752563662827015,
      "rewards/accuracy_reward": 0.0625,
      "rewards/cosine_scaled_reward": -0.13877009227871895,
      "step": 77
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3316.0833587646484,
      "epoch": 0.08914285714285715,
      "grad_norm": 0.04708974435925484,
      "kl": 3.929436206817627e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.374037332934512e-07,
      "loss": 0.0528,
      "reward": -0.1775032225996256,
      "reward_after_mean": -0.1775032225996256,
      "reward_after_std": 0.37231264263391495,
      "reward_before_mean": 0.06680050306022167,
      "reward_before_std": 0.3495578747242689,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2443037237972021,
      "reward_change_min": -0.40812752209603786,
      "reward_change_std": 0.15259934403002262,
      "reward_std": 0.3723126482218504,
      "rewards/accuracy_reward": 0.1458333395421505,
      "rewards/cosine_scaled_reward": -0.07903283275663853,
      "step": 78
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2158.958381652832,
      "epoch": 0.09028571428571429,
      "grad_norm": 0.09586747735738754,
      "kl": 3.816187381744385e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.2629708984760706e-07,
      "loss": -0.0522,
      "reward": -0.09330683806911111,
      "reward_after_mean": -0.09330683806911111,
      "reward_after_std": 0.5184203516691923,
      "reward_before_mean": 0.1577418657252565,
      "reward_before_std": 0.46990944538265467,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.25104869343340397,
      "reward_change_min": -0.45565746538341045,
      "reward_change_std": 0.16463573463261127,
      "reward_std": 0.5184203591197729,
      "rewards/accuracy_reward": 0.22916666977107525,
      "rewards/cosine_scaled_reward": -0.0714248213917017,
      "step": 79
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3231.041702270508,
      "epoch": 0.09142857142857143,
      "grad_norm": 0.057165395468473434,
      "kl": 5.085766315460205e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.1558482853517253e-07,
      "loss": 0.0002,
      "reward": -0.31745147332549095,
      "reward_after_mean": -0.31745147332549095,
      "reward_after_std": 0.3213096186518669,
      "reward_before_mean": -0.11810518335551023,
      "reward_before_std": 0.2888203295879066,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.1993462722748518,
      "reward_change_min": -0.3132110647857189,
      "reward_change_std": 0.12029875814914703,
      "reward_std": 0.3213096335530281,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.22227186150848866,
      "step": 80
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3167.083335876465,
      "epoch": 0.09257142857142857,
      "grad_norm": 0.06822940707206726,
      "kl": 5.334615707397461e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 2.0528000059645995e-07,
      "loss": -0.0023,
      "reward": -0.28903417102992535,
      "reward_after_mean": -0.28903417102992535,
      "reward_after_std": 0.36186219193041325,
      "reward_before_mean": -0.08435890730470419,
      "reward_before_std": 0.34423625422641635,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.20467526651918888,
      "reward_change_min": -0.3274262994527817,
      "reward_change_std": 0.1262927083298564,
      "reward_std": 0.3618621937930584,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.18852557707577944,
      "step": 81
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2622.916702270508,
      "epoch": 0.09371428571428571,
      "grad_norm": 0.0722210481762886,
      "kl": 4.655122756958008e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.9539516087697517e-07,
      "loss": 0.0292,
      "reward": -0.1543267703964375,
      "reward_after_mean": -0.1543267703964375,
      "reward_after_std": 0.5856532733887434,
      "reward_before_mean": 0.06535659916698933,
      "reward_before_std": 0.5598178403452039,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.219683350995183,
      "reward_change_min": -0.40967136807739735,
      "reward_change_std": 0.14763531927019358,
      "reward_std": 0.5856533050537109,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/cosine_scaled_reward": -0.08047674596309662,
      "step": 82
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2538.187515258789,
      "epoch": 0.09485714285714286,
      "grad_norm": 0.09561274945735931,
      "kl": 5.626678466796875e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.8594235253127372e-07,
      "loss": -0.037,
      "reward": -0.12532864138484,
      "reward_after_mean": -0.12532864138484,
      "reward_after_std": 0.4662170447409153,
      "reward_before_mean": 0.12323063798248768,
      "reward_before_std": 0.44423565454781055,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.24855928868055344,
      "reward_change_min": -0.4013477824628353,
      "reward_change_std": 0.15694511495530605,
      "reward_std": 0.4662170559167862,
      "rewards/accuracy_reward": 0.18750000558793545,
      "rewards/cosine_scaled_reward": -0.06426936015486717,
      "step": 83
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2805.375045776367,
      "epoch": 0.096,
      "grad_norm": 0.06256293505430222,
      "kl": 4.0903687477111816e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.7693309235023127e-07,
      "loss": 0.0204,
      "reward": 0.026594052091240883,
      "reward_after_mean": 0.026594052091240883,
      "reward_after_std": 0.6662927530705929,
      "reward_before_mean": 0.30712341889739037,
      "reward_before_std": 0.6726711131632328,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2805293593555689,
      "reward_change_min": -0.5315785314887762,
      "reward_change_std": 0.202547793276608,
      "reward_std": 0.6662927772849798,
      "rewards/accuracy_reward": 0.29166667722165585,
      "rewards/cosine_scaled_reward": 0.01545674353837967,
      "step": 84
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3143.3333740234375,
      "epoch": 0.09714285714285714,
      "grad_norm": 0.04833953082561493,
      "kl": 3.106147050857544e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.6837835672960831e-07,
      "loss": -0.0189,
      "reward": -0.2577883277554065,
      "reward_after_mean": -0.2577883277554065,
      "reward_after_std": 0.48671364039182663,
      "reward_before_mean": -0.06025014817714691,
      "reward_before_std": 0.4735073782503605,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.19753817282617092,
      "reward_change_min": -0.3981729503720999,
      "reward_change_std": 0.1415349431335926,
      "reward_std": 0.48671367578208447,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/cosine_scaled_reward": -0.16441681701689959,
      "step": 85
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2766.2708740234375,
      "epoch": 0.09828571428571428,
      "grad_norm": 0.07490651309490204,
      "kl": 5.46574592590332e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.6028856829700258e-07,
      "loss": -0.027,
      "reward": -0.2117796391248703,
      "reward_after_mean": -0.2117796391248703,
      "reward_after_std": 0.4173107650130987,
      "reward_before_mean": 0.01547178067266941,
      "reward_before_std": 0.416951060295105,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.22725142538547516,
      "reward_change_min": -0.4079761225730181,
      "reward_change_std": 0.15469573251903057,
      "reward_std": 0.41731079295277596,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/cosine_scaled_reward": -0.10952822491526604,
      "step": 86
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2761.729217529297,
      "epoch": 0.09942857142857142,
      "grad_norm": 0.07899302989244461,
      "kl": 4.908442497253418e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.5267358321348285e-07,
      "loss": -0.0014,
      "reward": -0.11103942012414336,
      "reward_after_mean": -0.11103942012414336,
      "reward_after_std": 0.4518774300813675,
      "reward_before_mean": 0.14697221852838993,
      "reward_before_std": 0.44158919900655746,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2580116242170334,
      "reward_change_min": -0.4169537350535393,
      "reward_change_std": 0.16394579596817493,
      "reward_std": 0.45187743939459324,
      "rewards/accuracy_reward": 0.18750000558793545,
      "rewards/cosine_scaled_reward": -0.04052778799086809,
      "step": 87
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2124.3125228881836,
      "epoch": 0.10057142857142858,
      "grad_norm": 0.09118274599313736,
      "kl": 5.093216896057129e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.4554267916537495e-07,
      "loss": 0.063,
      "reward": -0.03512320853769779,
      "reward_after_mean": -0.03512320853769779,
      "reward_after_std": 0.5551198851317167,
      "reward_before_mean": 0.22805100050754845,
      "reward_before_std": 0.4874242516234517,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2631742022931576,
      "reward_change_min": -0.46103948913514614,
      "reward_change_std": 0.16047734580934048,
      "reward_std": 0.5551198869943619,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/cosine_scaled_reward": -0.021949008107185364,
      "step": 88
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2926.0625610351562,
      "epoch": 0.10171428571428572,
      "grad_norm": 0.05478106066584587,
      "kl": 3.62396240234375e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.0017,
      "reward": -0.16949444822967052,
      "reward_after_mean": -0.16949444822967052,
      "reward_after_std": 0.5147665832191706,
      "reward_before_mean": 0.05882885679602623,
      "reward_before_std": 0.513655299320817,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2283233217895031,
      "reward_change_min": -0.4650176241993904,
      "reward_change_std": 0.16912943683564663,
      "reward_std": 0.5147665962576866,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/cosine_scaled_reward": -0.08700447157025337,
      "step": 89
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2371.3958435058594,
      "epoch": 0.10285714285714286,
      "grad_norm": 0.10766109824180603,
      "kl": 6.204843521118164e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.3276726544494571e-07,
      "loss": 0.0561,
      "reward": -0.3421127675101161,
      "reward_after_mean": -0.3421127675101161,
      "reward_after_std": 0.4089508708566427,
      "reward_before_mean": -0.17222417518496513,
      "reward_before_std": 0.3507535606622696,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.1698886025696993,
      "reward_change_min": -0.24556688778102398,
      "reward_change_std": 0.09127143956720829,
      "reward_std": 0.4089508727192879,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/cosine_scaled_reward": -0.21389084495604038,
      "step": 90
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2908.291679382324,
      "epoch": 0.104,
      "grad_norm": 0.0649840459227562,
      "kl": 5.263090133666992e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.2713832064634125e-07,
      "loss": 0.0049,
      "reward": -0.14402012154459953,
      "reward_after_mean": -0.14402012154459953,
      "reward_after_std": 0.3665870167315006,
      "reward_before_mean": 0.11315873265266418,
      "reward_before_std": 0.34818731527775526,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.25717885605990887,
      "reward_change_min": -0.40348367765545845,
      "reward_change_std": 0.15953873004764318,
      "reward_std": 0.3665870279073715,
      "rewards/accuracy_reward": 0.1875000074505806,
      "rewards/cosine_scaled_reward": -0.0743412896990776,
      "step": 91
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2475.2916946411133,
      "epoch": 0.10514285714285715,
      "grad_norm": 0.07881049811840057,
      "kl": 3.402680158615112e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.220245676671809e-07,
      "loss": -0.0348,
      "reward": -0.26179221691563725,
      "reward_after_mean": -0.26179221691563725,
      "reward_after_std": 0.4455073904246092,
      "reward_before_mean": -0.06640319153666496,
      "reward_before_std": 0.38197263330221176,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.19538902305066586,
      "reward_change_min": -0.28332172334194183,
      "reward_change_std": 0.10343744326382875,
      "reward_std": 0.4455074183642864,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/cosine_scaled_reward": -0.1289031896740198,
      "step": 92
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3584.0,
      "epoch": 0.10628571428571429,
      "grad_norm": 0.05205822363495827,
      "kl": 5.187094211578369e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.1743223682775649e-07,
      "loss": -0.0,
      "reward": -0.377890408039093,
      "reward_after_mean": -0.377890408039093,
      "reward_after_std": 0.305487222969532,
      "reward_before_mean": -0.20220234524458647,
      "reward_before_std": 0.26121661625802517,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.17568806931376457,
      "reward_change_min": -0.26276420429348946,
      "reward_change_std": 0.09641448874026537,
      "reward_std": 0.30548722483217716,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/cosine_scaled_reward": -0.22303568944334984,
      "step": 93
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2259.1250534057617,
      "epoch": 0.10742857142857143,
      "grad_norm": 0.07459476590156555,
      "kl": 5.307793617248535e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.1336692317580158e-07,
      "loss": 0.0457,
      "reward": -0.1999267702922225,
      "reward_after_mean": -0.1999267702922225,
      "reward_after_std": 0.3248994555324316,
      "reward_before_mean": 0.034845118410885334,
      "reward_before_std": 0.2530666273087263,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.23477189242839813,
      "reward_change_min": -0.33492102287709713,
      "reward_change_std": 0.12350414134562016,
      "reward_std": 0.32489946112036705,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/cosine_scaled_reward": -0.11098822485655546,
      "step": 94
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3346.750030517578,
      "epoch": 0.10857142857142857,
      "grad_norm": 0.05002405866980553,
      "kl": 4.0590763092041016e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.0983357966978745e-07,
      "loss": -0.0505,
      "reward": -0.27604287723079324,
      "reward_after_mean": -0.27604287723079324,
      "reward_after_std": 0.4140418618917465,
      "reward_before_mean": -0.07312451303005219,
      "reward_before_std": 0.4058398883789778,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.20291837118566036,
      "reward_change_min": -0.4037560001015663,
      "reward_change_std": 0.1442381963133812,
      "reward_std": 0.4140418656170368,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/cosine_scaled_reward": -0.15645784453954548,
      "step": 95
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2331.1041946411133,
      "epoch": 0.10971428571428571,
      "grad_norm": 0.07478164881467819,
      "kl": 3.7454068660736084e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.068365111445064e-07,
      "loss": 0.0203,
      "reward": -0.16277848929166794,
      "reward_after_mean": -0.16277848929166794,
      "reward_after_std": 0.4757649786770344,
      "reward_before_mean": 0.07331054843962193,
      "reward_before_std": 0.4654188547283411,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.23608904145658016,
      "reward_change_min": -0.38738977164030075,
      "reward_change_std": 0.15735210105776787,
      "reward_std": 0.4757649824023247,
      "rewards/accuracy_reward": 0.18750000558793545,
      "rewards/cosine_scaled_reward": -0.11418944969773293,
      "step": 96
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 3057.729217529297,
      "epoch": 0.11085714285714286,
      "grad_norm": 0.05840550735592842,
      "kl": 4.947185516357422e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0165,
      "reward": -0.1562044993042946,
      "reward_after_mean": -0.1562044993042946,
      "reward_after_std": 0.3517165407538414,
      "reward_before_mean": 0.09949065186083317,
      "reward_before_std": 0.33916947059333324,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.25569513998925686,
      "reward_change_min": -0.3923469968140125,
      "reward_change_std": 0.15778757631778717,
      "reward_std": 0.35171656124293804,
      "rewards/accuracy_reward": 0.1875000074505806,
      "rewards/cosine_scaled_reward": -0.08800935186445713,
      "step": 97
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2951.5833892822266,
      "epoch": 0.112,
      "grad_norm": 0.05241960659623146,
      "kl": 3.910064697265625e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.0246514708427701e-07,
      "loss": 0.0687,
      "reward": -0.1431019864976406,
      "reward_after_mean": -0.1431019864976406,
      "reward_after_std": 0.41360887698829174,
      "reward_before_mean": 0.09948256425559521,
      "reward_before_std": 0.33612028509378433,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.24258453957736492,
      "reward_change_min": -0.34666948951780796,
      "reward_change_std": 0.12914846278727055,
      "reward_std": 0.41360888816416264,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/cosine_scaled_reward": -0.06718412227928638,
      "step": 98
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2825.062526702881,
      "epoch": 0.11314285714285714,
      "grad_norm": 0.09673970937728882,
      "kl": 4.477798938751221e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.0109617738307911e-07,
      "loss": -0.0012,
      "reward": -0.14087136555463076,
      "reward_after_mean": -0.14087136555463076,
      "reward_after_std": 0.5487220250070095,
      "reward_before_mean": 0.08683701790869236,
      "reward_before_std": 0.5104697393253446,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.2277083769440651,
      "reward_change_min": -0.35002104938030243,
      "reward_change_std": 0.13233954645693302,
      "reward_std": 0.5487220343202353,
      "rewards/accuracy_reward": 0.16666667349636555,
      "rewards/cosine_scaled_reward": -0.07982964627444744,
      "step": 99
    },
    {
      "clip_fraction": 0.0,
      "completion_length": 2499.2083587646484,
      "epoch": 0.11428571428571428,
      "grad_norm": 0.07807345688343048,
      "kl": 5.307793617248535e-05,
      "lambda_div_used": 0.7000000000000001,
      "learning_rate": 1.002741278414069e-07,
      "loss": 0.0319,
      "reward": -0.19067936576902866,
      "reward_after_mean": -0.19067936576902866,
      "reward_after_std": 0.5915912743657827,
      "reward_before_mean": 0.013846036046743393,
      "reward_before_std": 0.5670791789889336,
      "reward_change_max": 0.0,
      "reward_change_mean": -0.20452538691461086,
      "reward_change_min": -0.4066152162849903,
      "reward_change_std": 0.14473197888582945,
      "reward_std": 0.5915912911295891,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/cosine_scaled_reward": -0.11115396713648806,
      "step": 100
    },
    {
      "epoch": 0.11428571428571428,
      "step": 100,
      "total_flos": 0.0,
      "train_loss": 0.004007526492219044,
      "train_runtime": 34518.5958,
      "train_samples_per_second": 0.139,
      "train_steps_per_second": 0.003
    }
  ],
  "logging_steps": 1,
  "max_steps": 100,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}