Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen2.5-1.5B-Instruct
-datasets: AI-MO/NuminaMath-TIR
 library_name: transformers
 model_name: Qwen2.5-1.5B-Open-R1-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen2.5-1.5B-Open-R1-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) on the [AI-MO/NuminaMath-TIR](https://huggingface.co/datasets/AI-MO/NuminaMath-TIR) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/hong-exploration/openr1/runs/plxee6ek) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/hong-exploration/openr1/runs/ax3fpq4v) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -5,9 +5,9 @@
     "eval_samples_per_second": 2.238,
     "eval_steps_per_second": 0.023,
     "total_flos": 0.0,
-    "train_loss": 0.08713241612633617,
-    "train_runtime": 26094.7605,
+    "train_loss": 0.11728219354807815,
+    "train_runtime": 26244.8941,
     "train_samples": 72441,
-    "train_samples_per_second": 2.776,
+    "train_samples_per_second": 2.76,
     "train_steps_per_second": 0.011
 }

config.json

@@ -23,7 +23,7 @@
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.49.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:503226a773136a687afaf4201ba575a9693f2ab24153e332c1274a962569cfa4
-size 3554214752
+oid sha256:b971812eb2b89634176d03de07ca3bebf63ed646850983bc29337f547b3b2cf5
+size 3087467144

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.08713241612633617,
-    "train_runtime": 26094.7605,
+    "train_loss": 0.11728219354807815,
+    "train_runtime": 26244.8941,
     "train_samples": 72441,
-    "train_samples_per_second": 2.776,
+    "train_samples_per_second": 2.76,
     "train_steps_per_second": 0.011
 }

trainer_state.json

@@ -9,774 +9,774 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 397.9532531738281,
+      "completion_length": 393.89309921264646,
       "epoch": 0.0176678445229682,
-      "grad_norm": 0.6871901154518127,
-      "kl": 0.0005688667297363281,
+      "grad_norm": 0.6810024386631965,
+      "kl": 0.0009959936141967773,
       "learning_rate": 3.448275862068966e-06,
       "loss": 0.0,
-      "reward": 0.6176339596509933,
-      "reward_std": 0.43154375404119494,
-      "rewards/accuracy_reward": 0.14810268627479672,
-      "rewards/format_reward": 0.4695312701165676,
+      "reward": 0.6294643148779869,
+      "reward_std": 0.43902995735406875,
+      "rewards/accuracy_reward": 0.14698661416769027,
+      "rewards/format_reward": 0.4824777036905289,
       "step": 5
     },
     {
-      "completion_length": 241.06094856262206,
+      "completion_length": 192.3870620727539,
       "epoch": 0.0353356890459364,
-      "grad_norm": 9.674223899841309,
-      "kl": 0.35438079833984376,
+      "grad_norm": 1.3096045466684565,
+      "kl": 0.5559478759765625,
       "learning_rate": 6.896551724137932e-06,
-      "loss": 0.0142,
-      "reward": 0.9390625447034836,
-      "reward_std": 0.28645481169223785,
-      "rewards/accuracy_reward": 0.07845982513390481,
-      "rewards/format_reward": 0.8606027245521546,
+      "loss": 0.0223,
+      "reward": 0.9539062917232514,
+      "reward_std": 0.24327411456033587,
+      "rewards/accuracy_reward": 0.06808036058209836,
+      "rewards/format_reward": 0.8858259305357933,
       "step": 10
     },
     {
-      "completion_length": 159.62165946960448,
+      "completion_length": 148.2626184463501,
       "epoch": 0.053003533568904596,
-      "grad_norm": 0.3432278037071228,
-      "kl": 0.055914306640625,
+      "grad_norm": 0.3220752359848403,
+      "kl": 0.0689697265625,
       "learning_rate": 1.0344827586206898e-05,
-      "loss": 0.0022,
-      "reward": 1.0266741544008255,
-      "reward_std": 0.2444406483322382,
-      "rewards/accuracy_reward": 0.09319196930155157,
-      "rewards/format_reward": 0.933482187986374,
+      "loss": 0.0028,
+      "reward": 1.0591518387198449,
+      "reward_std": 0.19840855058282614,
+      "rewards/accuracy_reward": 0.09352679075673223,
+      "rewards/format_reward": 0.9656250461935997,
       "step": 15
     },
     {
-      "completion_length": 106.59353141784668,
+      "completion_length": 148.3243366241455,
       "epoch": 0.0706713780918728,
-      "grad_norm": 0.3545452356338501,
-      "kl": 0.127545166015625,
+      "grad_norm": 0.2777358060320688,
+      "kl": 0.08660888671875,
       "learning_rate": 1.3793103448275863e-05,
-      "loss": 0.0051,
-      "reward": 1.0803571969270707,
-      "reward_std": 0.22546937856823207,
-      "rewards/accuracy_reward": 0.12232143525034189,
-      "rewards/format_reward": 0.9580357596278191,
+      "loss": 0.0035,
+      "reward": 1.0821429044008255,
+      "reward_std": 0.233628761023283,
+      "rewards/accuracy_reward": 0.12075893422588706,
+      "rewards/format_reward": 0.9613839760422707,
       "step": 20
     },
     {
-      "completion_length": 170.26206130981444,
+      "completion_length": 197.4337131500244,
       "epoch": 0.08833922261484099,
-      "grad_norm": 0.2860734164714813,
-      "kl": 0.10814208984375,
+      "grad_norm": 0.23923331052930114,
+      "kl": 0.10074462890625,
       "learning_rate": 1.7241379310344828e-05,
-      "loss": 0.0043,
-      "reward": 1.1982143491506576,
-      "reward_std": 0.27088444530963895,
-      "rewards/accuracy_reward": 0.22511161882430314,
-      "rewards/format_reward": 0.9731027260422707,
+      "loss": 0.004,
+      "reward": 1.1813616573810577,
+      "reward_std": 0.28772806450724603,
+      "rewards/accuracy_reward": 0.21294643841683863,
+      "rewards/format_reward": 0.968415217101574,
       "step": 25
     },
     {
-      "completion_length": 254.76976509094237,
+      "completion_length": 221.43449630737305,
       "epoch": 0.10600706713780919,
-      "grad_norm": 0.29131728410720825,
-      "kl": 2.9212158203125,
+      "grad_norm": 0.23678233280190517,
+      "kl": 0.1157470703125,
       "learning_rate": 1.999923511388017e-05,
-      "loss": 0.1172,
-      "reward": 1.187165230512619,
-      "reward_std": 0.35072063505649564,
-      "rewards/accuracy_reward": 0.2654017977416515,
-      "rewards/format_reward": 0.9217634350061417,
+      "loss": 0.0046,
+      "reward": 1.2284598618745803,
+      "reward_std": 0.28862305246293546,
+      "rewards/accuracy_reward": 0.26227679699659345,
+      "rewards/format_reward": 0.9661830767989159,
       "step": 30
     },
     {
-      "completion_length": 180.78471794128419,
+      "completion_length": 254.2150806427002,
       "epoch": 0.12367491166077739,
-      "grad_norm": 0.25441914796829224,
-      "kl": 0.14281005859375,
+      "grad_norm": 0.19756502476795068,
+      "kl": 0.1111083984375,
       "learning_rate": 1.9972476383747748e-05,
-      "loss": 0.0057,
-      "reward": 1.2573661237955094,
-      "reward_std": 0.27556890323758126,
-      "rewards/accuracy_reward": 0.27410715371370314,
-      "rewards/format_reward": 0.98325896859169,
+      "loss": 0.0044,
+      "reward": 1.2427455872297286,
+      "reward_std": 0.30822448506951333,
+      "rewards/accuracy_reward": 0.277790192887187,
+      "rewards/format_reward": 0.9649553999304772,
       "step": 35
     },
     {
-      "completion_length": 265.0565986633301,
+      "completion_length": 225.61786727905275,
       "epoch": 0.1413427561837456,
-      "grad_norm": 10500.8896484375,
-      "kl": 45.15205078125,
+      "grad_norm": 0.20692842581184812,
+      "kl": 2.31402587890625,
       "learning_rate": 1.9907590277344582e-05,
-      "loss": 1.8084,
-      "reward": 1.2706473737955093,
-      "reward_std": 0.307077270001173,
-      "rewards/accuracy_reward": 0.3092634052038193,
-      "rewards/format_reward": 0.9613839730620384,
+      "loss": 0.0926,
+      "reward": 1.2739955961704255,
+      "reward_std": 0.27363023944199083,
+      "rewards/accuracy_reward": 0.29486608430743216,
+      "rewards/format_reward": 0.9791295096278191,
       "step": 40
     },
     {
-      "completion_length": 177.18170375823973,
+      "completion_length": 237.31876029968262,
       "epoch": 0.15901060070671377,
-      "grad_norm": 11.025420188903809,
-      "kl": 0.8410888671875,
+      "grad_norm": 63.654079645433676,
+      "kl": 7.2471923828125,
       "learning_rate": 1.9804824871166254e-05,
-      "loss": 0.0336,
-      "reward": 1.197098270058632,
-      "reward_std": 0.3107326962053776,
-      "rewards/accuracy_reward": 0.2512276891618967,
-      "rewards/format_reward": 0.9458705827593803,
+      "loss": 0.2891,
+      "reward": 1.2812500596046448,
+      "reward_std": 0.3228338535875082,
+      "rewards/accuracy_reward": 0.33348215743899345,
+      "rewards/format_reward": 0.947767898440361,
       "step": 45
     },
     {
-      "completion_length": 197.19275417327881,
+      "completion_length": 298.5220016479492,
       "epoch": 0.17667844522968199,
-      "grad_norm": 2.2072925567626953,
-      "kl": 0.9974609375,
+      "grad_norm": 4.695169797433325,
+      "kl": 0.947802734375,
       "learning_rate": 1.9664573064143604e-05,
-      "loss": 0.0399,
-      "reward": 1.083928619325161,
-      "reward_std": 0.41752763763070105,
-      "rewards/accuracy_reward": 0.2433035835623741,
-      "rewards/format_reward": 0.8406250357627869,
+      "loss": 0.038,
+      "reward": 1.154687550663948,
+      "reward_std": 0.450396490842104,
+      "rewards/accuracy_reward": 0.3282366212457418,
+      "rewards/format_reward": 0.8264509290456772,
       "step": 50
     },
     {
-      "completion_length": 125.53616695404052,
+      "completion_length": 156.18192691802977,
       "epoch": 0.19434628975265017,
-      "grad_norm": 0.9618227481842041,
-      "kl": 0.76201171875,
+      "grad_norm": 1.7059064782094757,
+      "kl": 2.27958984375,
       "learning_rate": 1.948737107548771e-05,
-      "loss": 0.0305,
-      "reward": 1.118080411851406,
-      "reward_std": 0.3598920326679945,
-      "rewards/accuracy_reward": 0.2600446529686451,
-      "rewards/format_reward": 0.8580357536673546,
+      "loss": 0.0911,
+      "reward": 1.1735491514205934,
+      "reward_std": 0.3721325874328613,
+      "rewards/accuracy_reward": 0.28102679885923865,
+      "rewards/format_reward": 0.8925223663449288,
       "step": 55
     },
     {
-      "completion_length": 327.64934196472166,
+      "completion_length": 222.92512168884278,
       "epoch": 0.21201413427561838,
-      "grad_norm": 10.108561515808105,
-      "kl": 6.38251953125,
+      "grad_norm": 1.781183870462059,
+      "kl": 1.581396484375,
       "learning_rate": 1.9273896394584103e-05,
-      "loss": 0.2551,
-      "reward": 0.9488839730620384,
-      "reward_std": 0.4913848154246807,
-      "rewards/accuracy_reward": 0.24017858244478701,
-      "rewards/format_reward": 0.7087053894996643,
+      "loss": 0.0632,
+      "reward": 1.119531299173832,
+      "reward_std": 0.4729216001927853,
+      "rewards/accuracy_reward": 0.32868305072188375,
+      "rewards/format_reward": 0.7908482551574707,
       "step": 60
     },
     {
-      "completion_length": 242.94030113220214,
+      "completion_length": 255.30692977905272,
       "epoch": 0.22968197879858657,
-      "grad_norm": 2.333926200866699,
-      "kl": 1.8529296875,
+      "grad_norm": 1.9699387586342412,
+      "kl": 0.67763671875,
       "learning_rate": 1.9024965190774262e-05,
-      "loss": 0.0742,
-      "reward": 0.9114955708384513,
-      "reward_std": 0.5022768549621105,
-      "rewards/accuracy_reward": 0.23303572423756122,
-      "rewards/format_reward": 0.6784598529338837,
+      "loss": 0.0271,
+      "reward": 1.1059152275323867,
+      "reward_std": 0.4710687637329102,
+      "rewards/accuracy_reward": 0.3059151913970709,
+      "rewards/format_reward": 0.800000037252903,
       "step": 65
     },
     {
-      "completion_length": 415.55738792419436,
+      "completion_length": 302.30615234375,
       "epoch": 0.24734982332155478,
-      "grad_norm": 1.1008508205413818,
-      "kl": 11.3341796875,
+      "grad_norm": 5.642987581362726,
+      "kl": 0.4470703125,
       "learning_rate": 1.8741529192927528e-05,
-      "loss": 0.4533,
-      "reward": 0.920647357404232,
-      "reward_std": 0.4701464258134365,
-      "rewards/accuracy_reward": 0.21796876080334188,
-      "rewards/format_reward": 0.702678607404232,
+      "loss": 0.0179,
+      "reward": 1.1483259409666062,
+      "reward_std": 0.4444712668657303,
+      "rewards/accuracy_reward": 0.3253348361700773,
+      "rewards/format_reward": 0.822991107404232,
       "step": 70
     },
     {
-      "completion_length": 168.85536460876466,
+      "completion_length": 286.3686496734619,
       "epoch": 0.26501766784452296,
-      "grad_norm": 0.3387093245983124,
-      "kl": 0.340380859375,
+      "grad_norm": 68.54225530856624,
+      "kl": 0.7604248046875,
       "learning_rate": 1.8424672050733577e-05,
-      "loss": 0.0136,
-      "reward": 1.2478795111179353,
-      "reward_std": 0.2676353581249714,
-      "rewards/accuracy_reward": 0.26607143841683867,
-      "rewards/format_reward": 0.9818080767989159,
+      "loss": 0.0304,
+      "reward": 1.2591518491506577,
+      "reward_std": 0.3965926647186279,
+      "rewards/accuracy_reward": 0.36428573057055474,
+      "rewards/format_reward": 0.8948661148548126,
       "step": 75
     },
     {
-      "completion_length": 228.08896026611328,
+      "completion_length": 265.028918838501,
       "epoch": 0.2826855123674912,
-      "grad_norm": 0.9070476293563843,
-      "kl": 0.5783935546875,
+      "grad_norm": 9.929956785414381,
+      "kl": 0.63173828125,
       "learning_rate": 1.8075605191627242e-05,
-      "loss": 0.0231,
-      "reward": 1.2245536267757415,
-      "reward_std": 0.3449659403413534,
-      "rewards/accuracy_reward": 0.287500012665987,
-      "rewards/format_reward": 0.9370536103844642,
+      "loss": 0.0253,
+      "reward": 1.251116120815277,
+      "reward_std": 0.37127237394452095,
+      "rewards/accuracy_reward": 0.3388393010944128,
+      "rewards/format_reward": 0.9122768253087997,
       "step": 80
     },
     {
-      "completion_length": 349.13952407836916,
+      "completion_length": 239.63483276367188,
       "epoch": 0.3003533568904594,
-      "grad_norm": 3.210876703262329,
-      "kl": 2.055322265625,
+      "grad_norm": 13619.258585579173,
+      "kl": 63.713916015625,
       "learning_rate": 1.7695663189185703e-05,
-      "loss": 0.0823,
-      "reward": 1.1111607626080513,
-      "reward_std": 0.4750510863959789,
-      "rewards/accuracy_reward": 0.2958705469965935,
-      "rewards/format_reward": 0.8152902171015739,
+      "loss": 2.5585,
+      "reward": 1.220424148440361,
+      "reward_std": 0.37412303760647775,
+      "rewards/accuracy_reward": 0.30580358430743215,
+      "rewards/format_reward": 0.914620578289032,
       "step": 85
     },
     {
-      "completion_length": 356.24353942871096,
+      "completion_length": 252.38762359619142,
       "epoch": 0.31802120141342755,
-      "grad_norm": 0.17505526542663574,
-      "kl": 0.28251953125,
+      "grad_norm": 2.620982948598729,
+      "kl": 2.64697265625,
       "learning_rate": 1.7286298660705877e-05,
-      "loss": 0.0113,
-      "reward": 1.1301339864730835,
-      "reward_std": 0.49002460166811945,
-      "rewards/accuracy_reward": 0.32444198094308374,
-      "rewards/format_reward": 0.8056919991970062,
+      "loss": 0.1059,
+      "reward": 1.208258980512619,
+      "reward_std": 0.3845988120883703,
+      "rewards/accuracy_reward": 0.30401787124574187,
+      "rewards/format_reward": 0.9042411163449288,
       "step": 90
     },
     {
-      "completion_length": 333.82724838256837,
+      "completion_length": 240.73036766052246,
       "epoch": 0.33568904593639576,
-      "grad_norm": 0.16171219944953918,
-      "kl": 0.26669921875,
+      "grad_norm": 2.058492128617386,
+      "kl": 1.36826171875,
       "learning_rate": 1.6849076713469914e-05,
-      "loss": 0.0107,
-      "reward": 1.2251116514205933,
-      "reward_std": 0.3983605161309242,
-      "rewards/accuracy_reward": 0.33158483579754827,
-      "rewards/format_reward": 0.8935268297791481,
+      "loss": 0.0547,
+      "reward": 1.237500050663948,
+      "reward_std": 0.35002221167087555,
+      "rewards/accuracy_reward": 0.3074776943773031,
+      "rewards/format_reward": 0.9300223618745804,
       "step": 95
     },
     {
-      "completion_length": 238.18862686157226,
+      "completion_length": 223.18806838989258,
       "epoch": 0.35335689045936397,
-      "grad_norm": 1.341346263885498,
-      "kl": 0.214111328125,
+      "grad_norm": 24.446559888271214,
+      "kl": 2.938623046875,
       "learning_rate": 1.6385668960932143e-05,
-      "loss": 0.0086,
-      "reward": 1.2977679133415223,
-      "reward_std": 0.3174902819097042,
-      "rewards/accuracy_reward": 0.3381696566939354,
-      "rewards/format_reward": 0.9595982566475868,
+      "loss": 0.1176,
+      "reward": 1.3156250447034836,
+      "reward_std": 0.30762120597064496,
+      "rewards/accuracy_reward": 0.3568080525845289,
+      "rewards/format_reward": 0.9588170066475868,
       "step": 100
     },
     {
       "epoch": 0.35335689045936397,
-      "eval_completion_length": 224.875,
-      "eval_kl": 0.2919921875,
-      "eval_loss": 0.012192243710160255,
-      "eval_reward": 1.233258992433548,
-      "eval_reward_std": 0.3431517854332924,
-      "eval_rewards/accuracy_reward": 0.3002232313156128,
-      "eval_rewards/format_reward": 0.9330357611179352,
-      "eval_runtime": 37.6317,
-      "eval_samples_per_second": 2.631,
-      "eval_steps_per_second": 0.027,
+      "eval_completion_length": 193.20725631713867,
+      "eval_kl": 0.188232421875,
+      "eval_loss": 0.007562486920505762,
+      "eval_reward": 1.3404018580913544,
+      "eval_reward_std": 0.2854153998196125,
+      "eval_rewards/accuracy_reward": 0.352678582072258,
+      "eval_rewards/format_reward": 0.987723246216774,
+      "eval_runtime": 28.7786,
+      "eval_samples_per_second": 3.44,
+      "eval_steps_per_second": 0.035,
       "step": 100
     },
     {
-      "completion_length": 261.6398551940918,
+      "completion_length": 346.28260498046876,
       "epoch": 0.3710247349823322,
-      "grad_norm": 1.9713914394378662,
-      "kl": 0.8895751953125,
+      "grad_norm": 4864.347416469244,
+      "kl": 19.46124267578125,
       "learning_rate": 1.5897847131705194e-05,
-      "loss": 0.0356,
-      "reward": 1.1974330961704254,
-      "reward_std": 0.4034945048391819,
-      "rewards/accuracy_reward": 0.30926340594887736,
-      "rewards/format_reward": 0.8881696864962578,
+      "loss": 0.7786,
+      "reward": 1.175558090955019,
+      "reward_std": 0.3308975551277399,
+      "rewards/accuracy_reward": 0.33392858505249023,
+      "rewards/format_reward": 0.8416294999420643,
       "step": 105
     },
     {
-      "completion_length": 317.6125152587891,
+      "completion_length": 972.9336364746093,
       "epoch": 0.38869257950530034,
-      "grad_norm": 0.4451054632663727,
-      "kl": 1.036181640625,
+      "grad_norm": 2.2889203499644912,
+      "kl": 10.81435546875,
       "learning_rate": 1.5387476295779737e-05,
-      "loss": 0.0415,
-      "reward": 1.175223258137703,
-      "reward_std": 0.4380528204143047,
-      "rewards/accuracy_reward": 0.3095982287079096,
-      "rewards/format_reward": 0.865625037252903,
+      "loss": 0.4326,
+      "reward": 0.32176340855658053,
+      "reward_std": 0.4320876158773899,
+      "rewards/accuracy_reward": 0.17500000763684512,
+      "rewards/format_reward": 0.1467634005472064,
       "step": 110
     },
     {
-      "completion_length": 335.4704399108887,
+      "completion_length": 842.7776016235351,
       "epoch": 0.40636042402826855,
-      "grad_norm": 0.2159155160188675,
-      "kl": 0.2209228515625,
+      "grad_norm": 6.8529488082925685,
+      "kl": 1.0166015625,
       "learning_rate": 1.4856507733875837e-05,
-      "loss": 0.0088,
-      "reward": 1.2347098767757416,
-      "reward_std": 0.3829378016293049,
-      "rewards/accuracy_reward": 0.3314732301980257,
-      "rewards/format_reward": 0.9032366424798965,
+      "loss": 0.0407,
+      "reward": 0.7373884275555611,
+      "reward_std": 0.5035184837877751,
+      "rewards/accuracy_reward": 0.1222098263911903,
+      "rewards/format_reward": 0.6151785984635353,
       "step": 115
     },
     {
-      "completion_length": 326.20805130004885,
+      "completion_length": 547.6282608032227,
       "epoch": 0.42402826855123676,
-      "grad_norm": 1.607401728630066,
-      "kl": 0.16533203125,
+      "grad_norm": 0.19732296894180562,
+      "kl": 0.490673828125,
       "learning_rate": 1.4306971477188223e-05,
-      "loss": 0.0066,
-      "reward": 1.344531312584877,
-      "reward_std": 0.3044683691114187,
-      "rewards/accuracy_reward": 0.38002234175801275,
-      "rewards/format_reward": 0.9645089700818061,
+      "loss": 0.0196,
+      "reward": 1.0261161148548126,
+      "reward_std": 0.477869788557291,
+      "rewards/accuracy_reward": 0.21662947293370963,
+      "rewards/format_reward": 0.8094866469502449,
       "step": 120
     },
     {
-      "completion_length": 346.5212211608887,
+      "completion_length": 437.99120330810547,
       "epoch": 0.4416961130742049,
-      "grad_norm": 0.5397958755493164,
-      "kl": 0.2360595703125,
+      "grad_norm": 0.17418350781820638,
+      "kl": 0.356103515625,
       "learning_rate": 1.3740968546047935e-05,
-      "loss": 0.0094,
-      "reward": 1.3156250536441803,
-      "reward_std": 0.3454729124903679,
-      "rewards/accuracy_reward": 0.3767857324331999,
-      "rewards/format_reward": 0.9388393282890319,
+      "loss": 0.0142,
+      "reward": 1.2347098767757416,
+      "reward_std": 0.31738803870975973,
+      "rewards/accuracy_reward": 0.275781262293458,
+      "rewards/format_reward": 0.9589286133646965,
       "step": 125
     },
     {
-      "completion_length": 458.170223236084,
+      "completion_length": 285.84655113220214,
       "epoch": 0.45936395759717313,
-      "grad_norm": 2.1645452976226807,
-      "kl": 3.1985595703125,
+      "grad_norm": 0.1634304941360234,
+      "kl": 0.2971923828125,
       "learning_rate": 1.3160662917174045e-05,
-      "loss": 0.128,
-      "reward": 1.0997768297791481,
-      "reward_std": 0.5340591460466385,
-      "rewards/accuracy_reward": 0.3318080510944128,
-      "rewards/format_reward": 0.7679687827825546,
+      "loss": 0.0119,
+      "reward": 1.2758929073810577,
+      "reward_std": 0.306676396727562,
+      "rewards/accuracy_reward": 0.3169643014669418,
+      "rewards/format_reward": 0.9589286163449288,
       "step": 130
     },
     {
-      "completion_length": 381.4783645629883,
+      "completion_length": 316.3862846374512,
       "epoch": 0.47703180212014135,
-      "grad_norm": 251.16600036621094,
-      "kl": 4.827490234375,
+      "grad_norm": 0.16745415145225737,
+      "kl": 0.197900390625,
       "learning_rate": 1.2568273250226681e-05,
-      "loss": 0.1933,
-      "reward": 1.2683036237955094,
-      "reward_std": 0.38932468742132187,
-      "rewards/accuracy_reward": 0.3660714436322451,
-      "rewards/format_reward": 0.9022321864962578,
+      "loss": 0.0079,
+      "reward": 1.2949777275323868,
+      "reward_std": 0.2850029695779085,
+      "rewards/accuracy_reward": 0.32098215594887736,
+      "rewards/format_reward": 0.9739955782890319,
       "step": 135
     },
     {
-      "completion_length": 331.04789505004885,
+      "completion_length": 361.21407852172854,
       "epoch": 0.49469964664310956,
-      "grad_norm": 0.9643793702125549,
-      "kl": 0.825537109375,
+      "grad_norm": 0.3368945381876037,
+      "kl": 0.228125,
       "learning_rate": 1.1966064405292887e-05,
-      "loss": 0.033,
-      "reward": 1.3402902394533158,
-      "reward_std": 0.32668678648769855,
-      "rewards/accuracy_reward": 0.3882812689989805,
-      "rewards/format_reward": 0.95200896859169,
+      "loss": 0.0091,
+      "reward": 1.3011161178350448,
+      "reward_std": 0.31648399531841276,
+      "rewards/accuracy_reward": 0.33984376564621926,
+      "rewards/format_reward": 0.9612723648548126,
       "step": 140
     },
     {
-      "completion_length": 323.53405685424804,
+      "completion_length": 344.8656379699707,
       "epoch": 0.5123674911660777,
-      "grad_norm": 5.4946722984313965,
-      "kl": 1.706689453125,
+      "grad_norm": 11.170986788740064,
+      "kl": 0.554541015625,
       "learning_rate": 1.1356338783736256e-05,
-      "loss": 0.0683,
-      "reward": 1.3253348886966705,
-      "reward_std": 0.32204796373844147,
-      "rewards/accuracy_reward": 0.3709821619093418,
-      "rewards/format_reward": 0.9543527230620384,
+      "loss": 0.0222,
+      "reward": 1.2424107760190963,
+      "reward_std": 0.33078875951468945,
+      "rewards/accuracy_reward": 0.2889509070664644,
+      "rewards/format_reward": 0.9534598633646965,
       "step": 145
     },
     {
-      "completion_length": 334.69420700073243,
+      "completion_length": 520.6755836486816,
       "epoch": 0.5300353356890459,
-      "grad_norm": 28.387596130371094,
-      "kl": 2.99873046875,
+      "grad_norm": 0.40800837289998126,
+      "kl": 0.520849609375,
       "learning_rate": 1.0741427525516463e-05,
-      "loss": 0.12,
-      "reward": 1.272656312584877,
-      "reward_std": 0.3838721476495266,
-      "rewards/accuracy_reward": 0.3592634070664644,
-      "rewards/format_reward": 0.913392898440361,
+      "loss": 0.0208,
+      "reward": 1.1602679088711738,
+      "reward_std": 0.42941391095519066,
+      "rewards/accuracy_reward": 0.2982142973691225,
+      "rewards/format_reward": 0.8620536133646965,
       "step": 150
     },
     {
-      "completion_length": 350.9021369934082,
+      "completion_length": 519.8941070556641,
       "epoch": 0.5477031802120141,
-      "grad_norm": 6.340213298797607,
-      "kl": 1.597265625,
+      "grad_norm": 4.33328633720919,
+      "kl": 4.09052734375,
       "learning_rate": 1.012368159663363e-05,
-      "loss": 0.0639,
-      "reward": 0.9706473708152771,
-      "reward_std": 0.5445995971560478,
-      "rewards/accuracy_reward": 0.2651785839349031,
-      "rewards/format_reward": 0.7054687768220902,
+      "loss": 0.1637,
+      "reward": 0.9402902215719223,
+      "reward_std": 0.5052247367799282,
+      "rewards/accuracy_reward": 0.38359376713633536,
+      "rewards/format_reward": 0.5566964568570256,
       "step": 155
     },
     {
-      "completion_length": 307.3893013000488,
+      "completion_length": 473.456046295166,
       "epoch": 0.5653710247349824,
-      "grad_norm": 1.4934422969818115,
-      "kl": 1.1126953125,
+      "grad_norm": 34.64364006894349,
+      "kl": 2.496875,
       "learning_rate": 9.505462800772612e-06,
-      "loss": 0.0445,
-      "reward": 1.2178571939468383,
-      "reward_std": 0.4379304856061935,
-      "rewards/accuracy_reward": 0.33515626527369025,
-      "rewards/format_reward": 0.8827009320259094,
+      "loss": 0.1,
+      "reward": 0.9175223641097545,
+      "reward_std": 0.44062186665833,
+      "rewards/accuracy_reward": 0.4142857313156128,
+      "rewards/format_reward": 0.5032366307452321,
       "step": 160
     },
     {
-      "completion_length": 311.874341583252,
+      "completion_length": 357.35682373046876,
       "epoch": 0.5830388692579506,
-      "grad_norm": 1.8048169612884521,
-      "kl": 0.587841796875,
+      "grad_norm": 1.8671684175425967,
+      "kl": 1.8158203125,
       "learning_rate": 8.889134749511956e-06,
-      "loss": 0.0235,
-      "reward": 1.2705357730388642,
-      "reward_std": 0.4048257850110531,
-      "rewards/accuracy_reward": 0.3630580514669418,
-      "rewards/format_reward": 0.9074777245521546,
+      "loss": 0.0726,
+      "reward": 1.15245541036129,
+      "reward_std": 0.4938253708183765,
+      "rewards/accuracy_reward": 0.34151787087321284,
+      "rewards/format_reward": 0.8109375402331352,
       "step": 165
     },
     {
-      "completion_length": 340.8625144958496,
+      "completion_length": 424.50772018432616,
       "epoch": 0.6007067137809188,
-      "grad_norm": 11.069897651672363,
-      "kl": 0.925537109375,
+      "grad_norm": 157.5678868588516,
+      "kl": 6.02763671875,
       "learning_rate": 8.277053825620836e-06,
-      "loss": 0.037,
-      "reward": 1.2059152334928513,
-      "reward_std": 0.41814019829034804,
-      "rewards/accuracy_reward": 0.3325892988592386,
-      "rewards/format_reward": 0.8733259364962578,
+      "loss": 0.2413,
+      "reward": 1.0507812961935996,
+      "reward_std": 0.5250021070241928,
+      "rewards/accuracy_reward": 0.30535715594887736,
+      "rewards/format_reward": 0.7454241380095482,
       "step": 170
     },
     {
-      "completion_length": 396.6475631713867,
+      "completion_length": 461.17401657104494,
       "epoch": 0.6183745583038869,
-      "grad_norm": 2.382586717605591,
-      "kl": 2.17744140625,
+      "grad_norm": 4.550675721831043,
+      "kl": 9.0478515625,
       "learning_rate": 7.671560173993588e-06,
-      "loss": 0.0871,
-      "reward": 1.1248884528875351,
-      "reward_std": 0.5201080188155174,
-      "rewards/accuracy_reward": 0.32968751452863215,
-      "rewards/format_reward": 0.795200927555561,
+      "loss": 0.3619,
+      "reward": 0.8475446820259094,
+      "reward_std": 0.5652777068316936,
+      "rewards/accuracy_reward": 0.29564733542501925,
+      "rewards/format_reward": 0.551897344738245,
       "step": 175
     },
     {
-      "completion_length": 372.08874435424804,
+      "completion_length": 237.6184253692627,
       "epoch": 0.6360424028268551,
-      "grad_norm": 0.8225041031837463,
-      "kl": 1.33203125,
+      "grad_norm": 2.0935475462097988,
+      "kl": 1.74609375,
       "learning_rate": 7.07496875466589e-06,
-      "loss": 0.0533,
-      "reward": 1.198325940966606,
-      "reward_std": 0.46074943244457245,
-      "rewards/accuracy_reward": 0.3479910880327225,
-      "rewards/format_reward": 0.850334855914116,
+      "loss": 0.0698,
+      "reward": 1.217410758137703,
+      "reward_std": 0.36341911256313325,
+      "rewards/accuracy_reward": 0.31808037087321284,
+      "rewards/format_reward": 0.899330398440361,
       "step": 180
     },
     {
-      "completion_length": 355.8721153259277,
+      "completion_length": 225.6860595703125,
       "epoch": 0.6537102473498233,
-      "grad_norm": 0.28054162859916687,
-      "kl": 0.681640625,
+      "grad_norm": 2.8767733923026966,
+      "kl": 1.95068359375,
       "learning_rate": 6.489560492119225e-06,
-      "loss": 0.0273,
-      "reward": 1.2812500566244125,
-      "reward_std": 0.39972887486219405,
-      "rewards/accuracy_reward": 0.3756696589291096,
-      "rewards/format_reward": 0.9055803954601288,
+      "loss": 0.078,
+      "reward": 1.225446480512619,
+      "reward_std": 0.37394680455327034,
+      "rewards/accuracy_reward": 0.3206473350524902,
+      "rewards/format_reward": 0.9047991499304772,
       "step": 185
     },
     {
-      "completion_length": 368.4487907409668,
+      "completion_length": 231.34688453674318,
       "epoch": 0.6713780918727915,
-      "grad_norm": 1.2593623399734497,
-      "kl": 1.3046875,
+      "grad_norm": 2.2242779317938406,
+      "kl": 2.03359375,
       "learning_rate": 5.9175735547120975e-06,
-      "loss": 0.0522,
-      "reward": 1.2546875447034835,
-      "reward_std": 0.42497922107577324,
-      "rewards/accuracy_reward": 0.3743303745985031,
-      "rewards/format_reward": 0.8803571835160255,
+      "loss": 0.0813,
+      "reward": 1.1823661208152771,
+      "reward_std": 0.3818198271095753,
+      "rewards/accuracy_reward": 0.29419644065201284,
+      "rewards/format_reward": 0.8881696835160255,
       "step": 190
     },
     {
-      "completion_length": 373.5961112976074,
+      "completion_length": 248.6108371734619,
       "epoch": 0.6890459363957597,
-      "grad_norm": 0.49754491448402405,
-      "kl": 1.1783203125,
+      "grad_norm": 2.200150962540788,
+      "kl": 1.22763671875,
       "learning_rate": 5.361194797579108e-06,
-      "loss": 0.0471,
-      "reward": 1.207700952887535,
-      "reward_std": 0.4664508134126663,
-      "rewards/accuracy_reward": 0.35491073206067086,
-      "rewards/format_reward": 0.85279021859169,
+      "loss": 0.0491,
+      "reward": 1.1584821969270707,
+      "reward_std": 0.3713890131562948,
+      "rewards/accuracy_reward": 0.279241082444787,
+      "rewards/format_reward": 0.8792411088943481,
       "step": 195
     },
     {
-      "completion_length": 334.22434463500974,
+      "completion_length": 278.60481185913085,
       "epoch": 0.7067137809187279,
-      "grad_norm": 0.35578688979148865,
-      "kl": 0.481298828125,
+      "grad_norm": 1.9223755447710908,
+      "kl": 0.97314453125,
       "learning_rate": 4.8225514017138205e-06,
-      "loss": 0.0193,
-      "reward": 1.2475446999073028,
-      "reward_std": 0.41954655647277833,
-      "rewards/accuracy_reward": 0.354799123108387,
-      "rewards/format_reward": 0.8927455767989159,
+      "loss": 0.0389,
+      "reward": 1.137165230512619,
+      "reward_std": 0.35494174659252165,
+      "rewards/accuracy_reward": 0.2633928686380386,
+      "rewards/format_reward": 0.8737723618745804,
       "step": 200
     },
     {
       "epoch": 0.7067137809187279,
-      "eval_completion_length": 301.23802947998047,
-      "eval_kl": 0.3828125,
-      "eval_loss": 0.014114470221102238,
-      "eval_reward": 1.3214286267757416,
-      "eval_reward_std": 0.3996920883655548,
-      "eval_rewards/accuracy_reward": 0.4129464402794838,
-      "eval_rewards/format_reward": 0.9084821790456772,
-      "eval_runtime": 43.0662,
-      "eval_samples_per_second": 2.299,
-      "eval_steps_per_second": 0.023,
+      "eval_completion_length": 353.40673828125,
+      "eval_kl": 0.6904296875,
+      "eval_loss": 0.02955021895468235,
+      "eval_reward": 1.2142857909202576,
+      "eval_reward_std": 0.4049810841679573,
+      "eval_rewards/accuracy_reward": 0.3504464402794838,
+      "eval_rewards/format_reward": 0.8638393431901932,
+      "eval_runtime": 41.7526,
+      "eval_samples_per_second": 2.371,
+      "eval_steps_per_second": 0.024,
       "step": 200
     },
     {
-      "completion_length": 310.799568939209,
+      "completion_length": 301.07869415283204,
       "epoch": 0.7243816254416962,
-      "grad_norm": 0.23750554025173187,
-      "kl": 0.35625,
+      "grad_norm": 0.5851547376035416,
+      "kl": 0.7390625,
       "learning_rate": 4.303702741201431e-06,
-      "loss": 0.0143,
-      "reward": 1.285491129755974,
-      "reward_std": 0.4243697762489319,
-      "rewards/accuracy_reward": 0.38526787608861923,
-      "rewards/format_reward": 0.900223258137703,
+      "loss": 0.0296,
+      "reward": 1.1843750461935998,
+      "reward_std": 0.3787413276731968,
+      "rewards/accuracy_reward": 0.31361608356237414,
+      "rewards/format_reward": 0.8707589715719223,
       "step": 205
     },
     {
-      "completion_length": 308.95849609375,
+      "completion_length": 261.4496784210205,
       "epoch": 0.7420494699646644,
-      "grad_norm": 5.210378646850586,
-      "kl": 0.3337646484375,
+      "grad_norm": 0.3979385085158172,
+      "kl": 0.66767578125,
       "learning_rate": 3.8066325096949153e-06,
-      "loss": 0.0133,
-      "reward": 1.243192020058632,
-      "reward_std": 0.41392386630177497,
-      "rewards/accuracy_reward": 0.3464285880327225,
-      "rewards/format_reward": 0.8967634350061416,
+      "loss": 0.0267,
+      "reward": 1.1540179073810577,
+      "reward_std": 0.3514312542974949,
+      "rewards/accuracy_reward": 0.28270090483129023,
+      "rewards/format_reward": 0.8713170021772385,
       "step": 210
     },
     {
-      "completion_length": 302.08226890563964,
+      "completion_length": 313.5138542175293,
       "epoch": 0.7597173144876325,
-      "grad_norm": 76.71176147460938,
-      "kl": 6.380517578125,
+      "grad_norm": 0.5312090006838889,
+      "kl": 0.558251953125,
       "learning_rate": 3.3332411362372063e-06,
-      "loss": 0.2552,
-      "reward": 1.2647322028875352,
-      "reward_std": 0.4097910635173321,
-      "rewards/accuracy_reward": 0.36004466041922567,
-      "rewards/format_reward": 0.9046875461935997,
+      "loss": 0.0223,
+      "reward": 1.0902902379631996,
+      "reward_std": 0.40779144540429113,
+      "rewards/accuracy_reward": 0.27901787161827085,
+      "rewards/format_reward": 0.8112723603844643,
       "step": 215
     },
     {
-      "completion_length": 298.6371768951416,
+      "completion_length": 232.11842613220216,
       "epoch": 0.7773851590106007,
-      "grad_norm": 6.7656354904174805,
-      "kl": 1.5587890625,
+      "grad_norm": 0.4167441281663514,
+      "kl": 0.604345703125,
       "learning_rate": 2.8853385194256677e-06,
-      "loss": 0.0624,
-      "reward": 1.278571480512619,
-      "reward_std": 0.4139442786574364,
-      "rewards/accuracy_reward": 0.37700894586741923,
-      "rewards/format_reward": 0.9015625432133675,
+      "loss": 0.0242,
+      "reward": 1.1559152364730836,
+      "reward_std": 0.3387031778693199,
+      "rewards/accuracy_reward": 0.26361608169972894,
+      "rewards/format_reward": 0.8922991484403611,
       "step": 220
     },
     {
-      "completion_length": 297.99209213256836,
+      "completion_length": 267.01117248535155,
       "epoch": 0.7950530035335689,
-      "grad_norm": 4.592045307159424,
-      "kl": 2.12470703125,
+      "grad_norm": 0.4190976551497381,
+      "kl": 0.637890625,
       "learning_rate": 2.464637107698046e-06,
-      "loss": 0.085,
-      "reward": 1.267522367835045,
-      "reward_std": 0.40733090564608576,
-      "rewards/accuracy_reward": 0.36272323206067086,
-      "rewards/format_reward": 0.9047991514205933,
+      "loss": 0.0255,
+      "reward": 1.1452009439468385,
+      "reward_std": 0.3585247229784727,
+      "rewards/accuracy_reward": 0.27008929699659345,
+      "rewards/format_reward": 0.8751116409897804,
       "step": 225
     },
     {
-      "completion_length": 299.3953262329102,
+      "completion_length": 290.9562622070313,
       "epoch": 0.8127208480565371,
-      "grad_norm": 2.270449638366699,
-      "kl": 1.3138671875,
+      "grad_norm": 0.5115008967005016,
+      "kl": 0.59560546875,
       "learning_rate": 2.072745352195794e-06,
-      "loss": 0.0526,
-      "reward": 1.2822545230388642,
-      "reward_std": 0.4120013751089573,
-      "rewards/accuracy_reward": 0.37901787385344504,
-      "rewards/format_reward": 0.9032366499304771,
+      "loss": 0.0238,
+      "reward": 1.1270089790225029,
+      "reward_std": 0.4153431311249733,
+      "rewards/accuracy_reward": 0.2756696566939354,
+      "rewards/format_reward": 0.8513393253087997,
       "step": 230
     },
     {
-      "completion_length": 298.9074893951416,
+      "completion_length": 165.97846698760986,
       "epoch": 0.8303886925795053,
-      "grad_norm": 2.1383728981018066,
-      "kl": 1.87900390625,
+      "grad_norm": 0.39075163817088826,
+      "kl": 0.66572265625,
       "learning_rate": 1.7111615572361628e-06,
-      "loss": 0.0752,
-      "reward": 1.2869420260190965,
-      "reward_std": 0.4166914038360119,
-      "rewards/accuracy_reward": 0.3809151966124773,
-      "rewards/format_reward": 0.906026828289032,
+      "loss": 0.0266,
+      "reward": 1.1386161237955092,
+      "reward_std": 0.3385975301265717,
+      "rewards/accuracy_reward": 0.2363839393481612,
+      "rewards/format_reward": 0.9022321850061417,
       "step": 235
     },
     {
-      "completion_length": 305.2895233154297,
+      "completion_length": 212.52076835632323,
       "epoch": 0.8480565371024735,
-      "grad_norm": 1.2467355728149414,
-      "kl": 1.32548828125,
+      "grad_norm": 0.9590274026566307,
+      "kl": 0.64365234375,
       "learning_rate": 1.381268151904298e-06,
-      "loss": 0.053,
-      "reward": 1.2613839715719224,
-      "reward_std": 0.40479170605540277,
-      "rewards/accuracy_reward": 0.3544643003493547,
-      "rewards/format_reward": 0.9069196864962578,
+      "loss": 0.0258,
+      "reward": 1.1003348872065544,
+      "reward_std": 0.39529763013124464,
+      "rewards/accuracy_reward": 0.23895090408623218,
+      "rewards/format_reward": 0.8613839700818062,
       "step": 240
     },
     {
-      "completion_length": 299.97478942871095,
+      "completion_length": 247.38170852661133,
       "epoch": 0.8657243816254417,
-      "grad_norm": 1.4973329305648804,
-      "kl": 0.99462890625,
+      "grad_norm": 0.4221304613218481,
+      "kl": 0.7017578125,
       "learning_rate": 1.0843264046665558e-06,
-      "loss": 0.0398,
-      "reward": 1.2777902424335479,
-      "reward_std": 0.4032285138964653,
-      "rewards/accuracy_reward": 0.3697544813156128,
-      "rewards/format_reward": 0.9080357551574707,
+      "loss": 0.0281,
+      "reward": 1.0969866588711739,
+      "reward_std": 0.44245677813887596,
+      "rewards/accuracy_reward": 0.26350447721779346,
+      "rewards/format_reward": 0.8334821850061417,
       "step": 245
     },
     {
-      "completion_length": 294.9779163360596,
+      "completion_length": 217.14197425842286,
       "epoch": 0.8833922261484098,
-      "grad_norm": 0.6018723845481873,
-      "kl": 0.6970703125,
+      "grad_norm": 0.3590659050382048,
+      "kl": 0.712109375,
       "learning_rate": 8.214716012124491e-07,
-      "loss": 0.0279,
-      "reward": 1.2985491663217545,
-      "reward_std": 0.4030233770608902,
-      "rewards/accuracy_reward": 0.3878348398953676,
-      "rewards/format_reward": 0.9107143312692643,
+      "loss": 0.0285,
+      "reward": 1.1364955812692643,
+      "reward_std": 0.40216037854552267,
+      "rewards/accuracy_reward": 0.273214297555387,
+      "rewards/format_reward": 0.8632812887430191,
       "step": 250
     },
     {
-      "completion_length": 326.83907623291014,
+      "completion_length": 189.66574401855468,
       "epoch": 0.901060070671378,
-      "grad_norm": 0.3602770268917084,
-      "kl": 0.62119140625,
+      "grad_norm": 0.7601804769396449,
+      "kl": 0.76328125,
       "learning_rate": 5.937087039615619e-07,
-      "loss": 0.0249,
-      "reward": 1.228459867835045,
-      "reward_std": 0.43514843806624415,
-      "rewards/accuracy_reward": 0.3444196585565805,
-      "rewards/format_reward": 0.8840402200818062,
+      "loss": 0.0305,
+      "reward": 1.1437500655651092,
+      "reward_std": 0.37058388218283655,
+      "rewards/accuracy_reward": 0.2592634052038193,
+      "rewards/format_reward": 0.8844866499304771,
       "step": 255
     },
     {
-      "completion_length": 330.93282928466795,
+      "completion_length": 190.92657051086425,
       "epoch": 0.9187279151943463,
-      "grad_norm": 0.2966284155845642,
-      "kl": 0.5912109375,
+      "grad_norm": 0.3720093456894568,
+      "kl": 0.77138671875,
       "learning_rate": 4.019085098303077e-07,
-      "loss": 0.0237,
-      "reward": 1.2388393431901932,
-      "reward_std": 0.45445024222135544,
-      "rewards/accuracy_reward": 0.36796876713633536,
-      "rewards/format_reward": 0.8708705797791481,
+      "loss": 0.0308,
+      "reward": 1.164508980512619,
+      "reward_std": 0.37937103062868116,
+      "rewards/accuracy_reward": 0.2782366190105677,
+      "rewards/format_reward": 0.886272369325161,
       "step": 260
     },
     {
-      "completion_length": 342.0903045654297,
+      "completion_length": 187.0571517944336,
       "epoch": 0.9363957597173145,
-      "grad_norm": 0.3853587508201599,
-      "kl": 0.630712890625,
+      "grad_norm": 0.3119534949352162,
+      "kl": 0.762109375,
       "learning_rate": 2.4680432094837394e-07,
-      "loss": 0.0252,
-      "reward": 1.1949777334928513,
-      "reward_std": 0.4680465690791607,
-      "rewards/accuracy_reward": 0.3334821581840515,
-      "rewards/format_reward": 0.8614955797791481,
+      "loss": 0.0305,
+      "reward": 1.168638452887535,
+      "reward_std": 0.36167602315545083,
+      "rewards/accuracy_reward": 0.27533483430743216,
+      "rewards/format_reward": 0.893303619325161,
       "step": 265
     },
     {
-      "completion_length": 333.04599533081057,
+      "completion_length": 201.37422676086425,
       "epoch": 0.9540636042402827,
-      "grad_norm": 0.3707776665687561,
-      "kl": 0.55869140625,
+      "grad_norm": 0.6564897815580203,
+      "kl": 0.7783203125,
       "learning_rate": 1.289891410535593e-07,
-      "loss": 0.0223,
-      "reward": 1.2250000566244126,
-      "reward_std": 0.4637945219874382,
-      "rewards/accuracy_reward": 0.357142872735858,
-      "rewards/format_reward": 0.8678571864962578,
+      "loss": 0.0311,
+      "reward": 1.175446480512619,
+      "reward_std": 0.39644345417618754,
+      "rewards/accuracy_reward": 0.2938616216182709,
+      "rewards/format_reward": 0.8815848648548126,
       "step": 270
     },
     {
-      "completion_length": 335.9395240783691,
+      "completion_length": 195.3427547454834,
       "epoch": 0.9717314487632509,
-      "grad_norm": 0.2659561038017273,
-      "kl": 0.572705078125,
+      "grad_norm": 0.9880791982949485,
+      "kl": 0.77685546875,
       "learning_rate": 4.8913408283934874e-08,
-      "loss": 0.0229,
-      "reward": 1.216294702887535,
-      "reward_std": 0.44666091352701187,
-      "rewards/accuracy_reward": 0.3496651936322451,
-      "rewards/format_reward": 0.8666295036673546,
+      "loss": 0.0311,
+      "reward": 1.1718750536441802,
+      "reward_std": 0.3679466091096401,
+      "rewards/accuracy_reward": 0.2848214417695999,
+      "rewards/format_reward": 0.8870536148548126,
       "step": 275
     },
     {
-      "completion_length": 334.8103942871094,
+      "completion_length": 206.69465179443358,
       "epoch": 0.9893992932862191,
-      "grad_norm": 0.3593786060810089,
-      "kl": 0.59736328125,
+      "grad_norm": 0.521937304244144,
+      "kl": 0.7734375,
       "learning_rate": 6.883273035447335e-09,
-      "loss": 0.0239,
-      "reward": 1.2222098767757417,
-      "reward_std": 0.4681737795472145,
-      "rewards/accuracy_reward": 0.3558035880327225,
-      "rewards/format_reward": 0.8664062902331352,
+      "loss": 0.031,
+      "reward": 1.162388449907303,
+      "reward_std": 0.3875968180596828,
+      "rewards/accuracy_reward": 0.2827009055763483,
+      "rewards/format_reward": 0.8796875447034835,
       "step": 280
     },
     {
-      "completion_length": 347.41717020670575,
+      "completion_length": 208.12106704711914,
       "epoch": 1.0,
-      "kl": 0.5322265625,
-      "reward": 1.2327009389797847,
-      "reward_std": 0.44660818700989086,
-      "rewards/accuracy_reward": 0.3696056691308816,
-      "rewards/format_reward": 0.8630952735741934,
+      "kl": 0.7565104166666666,
+      "reward": 1.1662946939468384,
+      "reward_std": 0.4021851196885109,
+      "rewards/accuracy_reward": 0.2907366206248601,
+      "rewards/format_reward": 0.8755580733219782,
       "step": 283,
       "total_flos": 0.0,
-      "train_loss": 0.08713241612633617,
-      "train_runtime": 26094.7605,
-      "train_samples_per_second": 2.776,
+      "train_loss": 0.11728219354807815,
+      "train_runtime": 26244.8941,
+      "train_samples_per_second": 2.76,
       "train_steps_per_second": 0.011
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17f991b10f84285759eaffd9bc7353083daa3dce24c0d3c65bad841165cbe325
+oid sha256:aa6da04f477ec7b1381df2a074af29b3bc0803c1f1e33c08167200daa42863d1
 size 7544