Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen3-1.7B
-datasets: Blancy/1ktestfrom10kwithdifficultyclasses-without-difficult
 library_name: transformers
 model_name: Qwen3-1.7B-Open-R1-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen3-1.7B-Open-R1-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) on the [Blancy/1ktestfrom10kwithdifficultyclasses-without-difficult](https://huggingface.co/datasets/Blancy/1ktestfrom10kwithdifficultyclasses-without-difficult) dataset.
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/7kqetcgc) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/qvxw0mg7) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.004764942314540437,
-    "train_runtime": 16049.8199,
+    "train_loss": 0.2708671883675033,
+    "train_runtime": 44954.8681,
     "train_samples": 1000,
-    "train_samples_per_second": 0.062,
+    "train_samples_per_second": 0.067,
     "train_steps_per_second": 0.002
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.004764942314540437,
-    "train_runtime": 16049.8199,
+    "train_loss": 0.2708671883675033,
+    "train_runtime": 44954.8681,
     "train_samples": 1000,
-    "train_samples_per_second": 0.062,
+    "train_samples_per_second": 0.067,
     "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 36,
+  "global_step": 108,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -12,7 +12,7 @@
     {
       "completion_length": 3512.2977294921875,
       "epoch": 0.027972027972027972,
-      "grad_norm": 0.8605042695999146,
+      "grad_norm": 0.8601931929588318,
       "kl": 0.0,
       "learning_rate": 0.0,
       "loss": 0.0,
@@ -26,9 +26,9 @@
     {
       "completion_length": 3691.6666870117188,
       "epoch": 0.055944055944055944,
-      "grad_norm": 0.378974974155426,
+      "grad_norm": 0.37887123227119446,
       "kl": 0.0,
-      "learning_rate": 2.5e-06,
+      "learning_rate": 9.090909090909091e-07,
       "loss": 0.0,
       "reward": 0.4680059477686882,
       "reward_std": 0.1798743773251772,
@@ -38,495 +38,1503 @@
       "step": 2
     },
     {
-      "completion_length": 3411.9434814453125,
+      "completion_length": 3442.3929443359375,
       "epoch": 0.08391608391608392,
-      "grad_norm": 0.5123236775398254,
-      "kl": 0.000850677490234375,
-      "learning_rate": 5e-06,
+      "grad_norm": 0.6051873564720154,
+      "kl": 0.0008230209350585938,
+      "learning_rate": 1.8181818181818183e-06,
       "loss": 0.0,
-      "reward": 0.5870535895228386,
-      "reward_std": 0.15364255383610725,
-      "rewards/accuracy_reward": 0.15773810120299459,
+      "reward": 0.5766369253396988,
+      "reward_std": 0.12715193443000317,
+      "rewards/accuracy_reward": 0.1547619104385376,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4293154776096344,
+      "rewards/tag_count_reward": 0.4218750074505806,
       "step": 3
     },
     {
-      "completion_length": 3326.3631591796875,
+      "completion_length": 3409.6935424804688,
       "epoch": 0.11188811188811189,
-      "grad_norm": 0.6567738056182861,
-      "kl": 0.0010671615600585938,
-      "learning_rate": 7.500000000000001e-06,
+      "grad_norm": 1.0704518556594849,
+      "kl": 0.0009069442749023438,
+      "learning_rate": 2.7272727272727272e-06,
       "loss": 0.0,
-      "reward": 0.5238095447421074,
-      "reward_std": 0.12606673501431942,
-      "rewards/accuracy_reward": 0.11309523973613977,
+      "reward": 0.5037202537059784,
+      "reward_std": 0.12088673282414675,
+      "rewards/accuracy_reward": 0.10416666883975267,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4107142984867096,
+      "rewards/tag_count_reward": 0.3995535746216774,
       "step": 4
     },
     {
-      "completion_length": 3617.0536499023438,
+      "completion_length": 3750.7233276367188,
       "epoch": 0.13986013986013987,
-      "grad_norm": 0.5138365030288696,
-      "kl": 0.00595855712890625,
-      "learning_rate": 1e-05,
-      "loss": 0.0002,
-      "reward": 0.5572916716337204,
-      "reward_std": 0.17760900035500526,
-      "rewards/accuracy_reward": 0.13690476398915052,
+      "grad_norm": 1.2529385089874268,
+      "kl": 0.001476287841796875,
+      "learning_rate": 3.6363636363636366e-06,
+      "loss": 0.0001,
+      "reward": 0.495535708963871,
+      "reward_std": 0.149957662448287,
+      "rewards/accuracy_reward": 0.12202381482347846,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4203869104385376,
+      "rewards/tag_count_reward": 0.373511902987957,
       "step": 5
     },
     {
-      "completion_length": 3649.2351684570312,
+      "completion_length": 3797.9166870117188,
       "epoch": 0.16783216783216784,
-      "grad_norm": 10.930187225341797,
-      "kl": 0.11383056640625,
-      "learning_rate": 9.978331270024887e-06,
-      "loss": 0.0045,
-      "reward": 0.5364583432674408,
-      "reward_std": 0.24191416427493095,
-      "rewards/accuracy_reward": 0.05952381109818816,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4769345298409462,
+      "grad_norm": 0.43379437923431396,
+      "kl": 0.00420379638671875,
+      "learning_rate": 4.5454545454545455e-06,
+      "loss": 0.0002,
+      "reward": 0.4501488134264946,
+      "reward_std": 0.15580809116363525,
+      "rewards/accuracy_reward": 0.07738095289096236,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3727678656578064,
       "step": 6
     },
     {
-      "completion_length": 3823.256103515625,
+      "completion_length": 3904.8424072265625,
       "epoch": 0.1958041958041958,
-      "grad_norm": 0.22635069489479065,
-      "kl": 0.02630615234375,
-      "learning_rate": 9.913533761814537e-06,
-      "loss": 0.0011,
-      "reward": 0.5312499925494194,
-      "reward_std": 0.16136238723993301,
-      "rewards/accuracy_reward": 0.14880952797830105,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.382440485060215,
+      "grad_norm": 0.2574431300163269,
+      "kl": 0.0105133056640625,
+      "learning_rate": 5.4545454545454545e-06,
+      "loss": 0.0004,
+      "reward": 0.4895833358168602,
+      "reward_std": 0.18993620201945305,
+      "rewards/accuracy_reward": 0.11607143236324191,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.373511902987957,
       "step": 7
     },
     {
-      "completion_length": 3570.291748046875,
+      "completion_length": 3614.2798461914062,
       "epoch": 0.22377622377622378,
-      "grad_norm": 0.19483903050422668,
-      "kl": 0.03973388671875,
-      "learning_rate": 9.80623151079494e-06,
-      "loss": 0.0016,
-      "reward": 0.4813988208770752,
-      "reward_std": 0.15597185026854277,
-      "rewards/accuracy_reward": 0.09226190880872309,
+      "grad_norm": 0.6184343695640564,
+      "kl": 0.018280029296875,
+      "learning_rate": 6.363636363636364e-06,
+      "loss": 0.0007,
+      "reward": 0.524553582072258,
+      "reward_std": 0.20281112380325794,
+      "rewards/accuracy_reward": 0.09226190717890859,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3891369104385376,
+      "rewards/tag_count_reward": 0.432291679084301,
       "step": 8
     },
     {
-      "completion_length": 3669.8363647460938,
+      "completion_length": 3627.4553833007812,
       "epoch": 0.2517482517482518,
-      "grad_norm": 0.3098791539669037,
-      "kl": 0.0445556640625,
-      "learning_rate": 9.65745789630079e-06,
-      "loss": 0.0018,
-      "reward": 0.5543154776096344,
-      "reward_std": 0.1814326737076044,
-      "rewards/accuracy_reward": 0.16369047993794084,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3906250149011612,
+      "grad_norm": 0.2969047725200653,
+      "kl": 0.024139404296875,
+      "learning_rate": 7.272727272727273e-06,
+      "loss": 0.001,
+      "reward": 0.5773809626698494,
+      "reward_std": 0.21523480489850044,
+      "rewards/accuracy_reward": 0.16369048040360212,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4136904776096344,
       "step": 9
     },
     {
-      "completion_length": 3507.3483276367188,
+      "completion_length": 3487.21435546875,
       "epoch": 0.27972027972027974,
-      "grad_norm": 0.23944704234600067,
-      "kl": 0.04766845703125,
-      "learning_rate": 9.468645689567599e-06,
-      "loss": 0.0019,
-      "reward": 0.6175595223903656,
-      "reward_std": 0.20797590538859367,
-      "rewards/accuracy_reward": 0.1994047649204731,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.418154776096344,
+      "grad_norm": 0.24078179895877838,
+      "kl": 0.024017333984375,
+      "learning_rate": 8.181818181818183e-06,
+      "loss": 0.001,
+      "reward": 0.6346726417541504,
+      "reward_std": 0.2543078139424324,
+      "rewards/accuracy_reward": 0.1904761977493763,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4441964402794838,
       "step": 10
     },
     {
-      "completion_length": 3486.0625610351562,
+      "completion_length": 3455.8631591796875,
       "epoch": 0.3076923076923077,
-      "grad_norm": 0.2252836674451828,
-      "kl": 0.0560302734375,
-      "learning_rate": 9.241613255361455e-06,
-      "loss": 0.0022,
-      "reward": 0.5111607238650322,
-      "reward_std": 0.14905241318047047,
-      "rewards/accuracy_reward": 0.1250000037252903,
+      "grad_norm": 0.4435509741306305,
+      "kl": 0.033843994140625,
+      "learning_rate": 9.090909090909091e-06,
+      "loss": 0.0014,
+      "reward": 0.5729166865348816,
+      "reward_std": 0.18820499069988728,
+      "rewards/accuracy_reward": 0.14285714365541935,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3861607164144516,
+      "rewards/tag_count_reward": 0.4300595372915268,
       "step": 11
     },
     {
-      "completion_length": 3626.5327758789062,
+      "completion_length": 3493.541748046875,
       "epoch": 0.3356643356643357,
-      "grad_norm": 0.3740616738796234,
-      "kl": 0.06890869140625,
-      "learning_rate": 8.978547040132317e-06,
-      "loss": 0.0028,
-      "reward": 0.4546131119132042,
-      "reward_std": 0.10877569299191236,
-      "rewards/accuracy_reward": 0.07142857369035482,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3831845298409462,
+      "grad_norm": 0.23296485841274261,
+      "kl": 0.0333251953125,
+      "learning_rate": 1e-05,
+      "loss": 0.0013,
+      "reward": 0.5052083358168602,
+      "reward_std": 0.16948581114411354,
+      "rewards/accuracy_reward": 0.09821428847499192,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4069940596818924,
       "step": 12
     },
     {
-      "completion_length": 3609.3452758789062,
+      "completion_length": 3424.3751220703125,
       "epoch": 0.36363636363636365,
-      "grad_norm": 0.24548310041427612,
-      "kl": 0.073486328125,
-      "learning_rate": 8.681980515339464e-06,
-      "loss": 0.0029,
-      "reward": 0.7008928880095482,
-      "reward_std": 0.1800659578293562,
-      "rewards/accuracy_reward": 0.3184523917734623,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.382440485060215,
+      "grad_norm": 131.0524139404297,
+      "kl": 0.958984375,
+      "learning_rate": 9.997640060704818e-06,
+      "loss": 0.0384,
+      "reward": 0.7269345447421074,
+      "reward_std": 0.20091417245566845,
+      "rewards/accuracy_reward": 0.3273809589445591,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3995535746216774,
       "step": 13
     },
     {
-      "completion_length": 3744.2798461914062,
+      "completion_length": 3556.96435546875,
       "epoch": 0.3916083916083916,
-      "grad_norm": 0.2982732057571411,
-      "kl": 0.0880126953125,
-      "learning_rate": 8.354769778736407e-06,
-      "loss": 0.0035,
-      "reward": 0.7291666865348816,
-      "reward_std": 0.3022393621504307,
-      "rewards/accuracy_reward": 0.3660714328289032,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3630952388048172,
+      "grad_norm": 0.4128544330596924,
+      "kl": 0.051025390625,
+      "learning_rate": 9.990562718069703e-06,
+      "loss": 0.002,
+      "reward": 0.7678571492433548,
+      "reward_std": 0.29729820415377617,
+      "rewards/accuracy_reward": 0.3750000186264515,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3928571492433548,
       "step": 14
     },
     {
-      "completion_length": 4129.886962890625,
+      "completion_length": 3937.729248046875,
       "epoch": 0.4195804195804196,
-      "grad_norm": 0.21393992006778717,
-      "kl": 0.09912109375,
-      "learning_rate": 8.00006604858821e-06,
-      "loss": 0.004,
-      "reward": 0.5193452388048172,
-      "reward_std": 0.18354132398962975,
-      "rewards/accuracy_reward": 0.20833333395421505,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3110119104385376,
+      "grad_norm": 0.3086971938610077,
+      "kl": 0.0550537109375,
+      "learning_rate": 9.978775395249763e-06,
+      "loss": 0.0022,
+      "reward": 0.5550595372915268,
+      "reward_std": 0.16711290925741196,
+      "rewards/accuracy_reward": 0.2291666716337204,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3258928656578064,
       "step": 15
     },
     {
-      "completion_length": 4105.565612792969,
+      "completion_length": 3943.59228515625,
       "epoch": 0.44755244755244755,
-      "grad_norm": 0.2648344337940216,
-      "kl": 0.111572265625,
-      "learning_rate": 7.621285315716991e-06,
-      "loss": 0.0045,
-      "reward": 0.5394345372915268,
-      "reward_std": 0.20268973521888256,
-      "rewards/accuracy_reward": 0.22023809514939785,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3191964328289032,
+      "grad_norm": 0.543189525604248,
+      "kl": 0.064453125,
+      "learning_rate": 9.962290455518914e-06,
+      "loss": 0.0026,
+      "reward": 0.584077388048172,
+      "reward_std": 0.2332012578845024,
+      "rewards/accuracy_reward": 0.24107143003493547,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3430059626698494,
       "step": 16
     },
     {
-      "completion_length": 4122.520812988281,
+      "completion_length": 4002.994140625,
       "epoch": 0.4755244755244755,
-      "grad_norm": 0.18669383227825165,
-      "kl": 0.135986328125,
-      "learning_rate": 7.222075445642904e-06,
-      "loss": 0.0054,
-      "reward": 0.538690485060215,
-      "reward_std": 0.12965587340295315,
-      "rewards/accuracy_reward": 0.22619048180058599,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3125000074505806,
+      "grad_norm": 0.17314046621322632,
+      "kl": 0.08203125,
+      "learning_rate": 9.941125189302508e-06,
+      "loss": 0.0033,
+      "reward": 0.5959821566939354,
+      "reward_std": 0.18275887705385685,
+      "rewards/accuracy_reward": 0.2619047621265054,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3340773805975914,
       "step": 17
     },
     {
-      "completion_length": 4098.357116699219,
+      "completion_length": 4070.3750610351562,
       "epoch": 0.5034965034965035,
-      "grad_norm": 0.12457378208637238,
-      "kl": 0.130126953125,
-      "learning_rate": 6.80628104764508e-06,
-      "loss": 0.0052,
-      "reward": 0.5104166865348816,
-      "reward_std": 0.1256360486149788,
-      "rewards/accuracy_reward": 0.1994047632906586,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3110119104385376,
+      "grad_norm": 0.18910881876945496,
+      "kl": 0.0848388671875,
+      "learning_rate": 9.915301796042076e-06,
+      "loss": 0.0034,
+      "reward": 0.5171131044626236,
+      "reward_std": 0.16867330251261592,
+      "rewards/accuracy_reward": 0.19940475933253765,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3177083358168602,
       "step": 18
     },
     {
-      "completion_length": 3781.9761962890625,
+      "completion_length": 3723.9761962890625,
       "epoch": 0.5314685314685315,
-      "grad_norm": 0.1671716421842575,
-      "kl": 0.1495361328125,
-      "learning_rate": 6.377906449072578e-06,
-      "loss": 0.006,
-      "reward": 0.70238097012043,
-      "reward_std": 0.2050741408020258,
-      "rewards/accuracy_reward": 0.354166679084301,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.348214291036129,
+      "grad_norm": 0.233817920088768,
+      "kl": 0.1026611328125,
+      "learning_rate": 9.884847360911168e-06,
+      "loss": 0.0041,
+      "reward": 0.7380952388048172,
+      "reward_std": 0.1809748988598585,
+      "rewards/accuracy_reward": 0.377976194024086,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3601190596818924,
       "step": 19
     },
     {
-      "completion_length": 4104.199401855469,
+      "completion_length": 4125.550720214844,
       "epoch": 0.5594405594405595,
-      "grad_norm": 0.12906776368618011,
-      "kl": 0.164306640625,
-      "learning_rate": 5.9410771314830255e-06,
-      "loss": 0.0066,
-      "reward": 0.4784226268529892,
-      "reward_std": 0.11611867044121027,
-      "rewards/accuracy_reward": 0.18154762499034405,
+      "grad_norm": 0.12808525562286377,
+      "kl": 0.12451171875,
+      "learning_rate": 9.849793826406752e-06,
+      "loss": 0.005,
+      "reward": 0.4702381044626236,
+      "reward_std": 0.15858174674212933,
+      "rewards/accuracy_reward": 0.17261905409395695,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.296875,
+      "rewards/tag_count_reward": 0.2976190596818924,
       "step": 20
     },
     {
-      "completion_length": 4042.2559814453125,
+      "completion_length": 4082.4910888671875,
       "epoch": 0.5874125874125874,
-      "grad_norm": 0.26009050011634827,
-      "kl": 0.157470703125,
-      "learning_rate": 5.500000000000001e-06,
-      "loss": 0.0063,
-      "reward": 0.5096726417541504,
-      "reward_std": 0.19976547360420227,
-      "rewards/accuracy_reward": 0.19642857555299997,
+      "grad_norm": 0.2625791132450104,
+      "kl": 0.1243896484375,
+      "learning_rate": 9.810177958845942e-06,
+      "loss": 0.005,
+      "reward": 0.494791679084301,
+      "reward_std": 0.1791169587522745,
+      "rewards/accuracy_reward": 0.18154762033373117,
       "rewards/format_reward": 0.0,
       "rewards/tag_count_reward": 0.3132440522313118,
       "step": 21
     },
     {
-      "completion_length": 4085.7352905273438,
+      "completion_length": 4112.172607421875,
       "epoch": 0.6153846153846154,
-      "grad_norm": 0.1436339169740677,
-      "kl": 0.16455078125,
-      "learning_rate": 5.0589228685169776e-06,
-      "loss": 0.0066,
-      "reward": 0.5535714402794838,
-      "reward_std": 0.15361771546304226,
-      "rewards/accuracy_reward": 0.2380952388048172,
+      "grad_norm": 0.2053973376750946,
+      "kl": 0.13818359375,
+      "learning_rate": 9.766041309803218e-06,
+      "loss": 0.0055,
+      "reward": 0.5200892984867096,
+      "reward_std": 0.11136791668832302,
+      "rewards/accuracy_reward": 0.21130952797830105,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.315476194024086,
+      "rewards/tag_count_reward": 0.3087797611951828,
       "step": 22
     },
     {
-      "completion_length": 3802.232177734375,
+      "completion_length": 3917.4761962890625,
       "epoch": 0.6433566433566433,
-      "grad_norm": 0.5061047673225403,
-      "kl": 0.18212890625,
-      "learning_rate": 4.622093550927423e-06,
-      "loss": 0.0073,
-      "reward": 0.693452388048172,
-      "reward_std": 0.22774529829621315,
-      "rewards/accuracy_reward": 0.3511904813349247,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3422619104385376,
+      "grad_norm": 0.19135095179080963,
+      "kl": 0.160400390625,
+      "learning_rate": 9.717430172528548e-06,
+      "loss": 0.0064,
+      "reward": 0.6279762163758278,
+      "reward_std": 0.19610902667045593,
+      "rewards/accuracy_reward": 0.2976190522313118,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3303571417927742,
       "step": 23
     },
     {
-      "completion_length": 3550.8185424804688,
+      "completion_length": 3670.77099609375,
       "epoch": 0.6713286713286714,
-      "grad_norm": 0.14333468675613403,
-      "kl": 0.171875,
-      "learning_rate": 4.193718952354921e-06,
-      "loss": 0.0069,
-      "reward": 0.7083333432674408,
-      "reward_std": 0.1849147491157055,
-      "rewards/accuracy_reward": 0.3511904813349247,
+      "grad_norm": 0.16754572093486786,
+      "kl": 0.16552734375,
+      "learning_rate": 9.66439553339217e-06,
+      "loss": 0.0066,
+      "reward": 0.67113097012043,
+      "reward_std": 0.1715894378721714,
+      "rewards/accuracy_reward": 0.3244047649204731,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3571428582072258,
+      "rewards/tag_count_reward": 0.3467261865735054,
       "step": 24
     },
     {
-      "completion_length": 3492.514892578125,
+      "completion_length": 3596.1072387695312,
       "epoch": 0.6993006993006993,
-      "grad_norm": 0.14843598008155823,
-      "kl": 0.174072265625,
-      "learning_rate": 3.777924554357096e-06,
-      "loss": 0.007,
-      "reward": 0.8318452537059784,
-      "reward_std": 0.24168619140982628,
-      "rewards/accuracy_reward": 0.4494047686457634,
+      "grad_norm": 0.2030419558286667,
+      "kl": 0.1728515625,
+      "learning_rate": 9.606993018406931e-06,
+      "loss": 0.0069,
+      "reward": 0.7254464328289032,
+      "reward_std": 0.26019805669784546,
+      "rewards/accuracy_reward": 0.351190485060215,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.382440485060215,
+      "rewards/tag_count_reward": 0.3742559626698494,
       "step": 25
     },
     {
-      "completion_length": 3533.2828369140625,
+      "completion_length": 3640.6488037109375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.13751864433288574,
-      "kl": 0.18212890625,
-      "learning_rate": 3.378714684283011e-06,
-      "loss": 0.0073,
-      "reward": 0.737351194024086,
-      "reward_std": 0.16352506168186665,
-      "rewards/accuracy_reward": 0.380952388048172,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3563988134264946,
+      "grad_norm": 61.024208068847656,
+      "kl": 1.417724609375,
+      "learning_rate": 9.54528283488428e-06,
+      "loss": 0.0568,
+      "reward": 0.7016369253396988,
+      "reward_std": 0.12191555928438902,
+      "rewards/accuracy_reward": 0.3571428656578064,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3444940522313118,
       "step": 26
     },
     {
-      "completion_length": 3208.8630981445312,
+      "completion_length": 3402.327392578125,
       "epoch": 0.7552447552447552,
-      "grad_norm": 0.47788581252098083,
-      "kl": 0.20458984375,
-      "learning_rate": 2.9999339514117913e-06,
-      "loss": 0.0082,
-      "reward": 0.7946428656578064,
-      "reward_std": 0.18282870575785637,
-      "rewards/accuracy_reward": 0.3898809514939785,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4047619104385376,
+      "grad_norm": 0.19034799933433533,
+      "kl": 0.213623046875,
+      "learning_rate": 9.479329708285107e-06,
+      "loss": 0.0085,
+      "reward": 0.7440476268529892,
+      "reward_std": 0.22099306993186474,
+      "rewards/accuracy_reward": 0.3482142873108387,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3958333432674408,
       "step": 27
     },
     {
-      "completion_length": 3646.8929443359375,
+      "completion_length": 3721.1815795898438,
       "epoch": 0.7832167832167832,
-      "grad_norm": 0.1616424322128296,
-      "kl": 0.191162109375,
-      "learning_rate": 2.645230221263596e-06,
-      "loss": 0.0076,
-      "reward": 0.5208333432674408,
-      "reward_std": 0.13056958466768265,
-      "rewards/accuracy_reward": 0.16071428847499192,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3601190522313118,
+      "grad_norm": 0.41007938981056213,
+      "kl": 0.23095703125,
+      "learning_rate": 9.40920281433168e-06,
+      "loss": 0.0092,
+      "reward": 0.4918154776096344,
+      "reward_std": 0.14000625722110271,
+      "rewards/accuracy_reward": 0.13988095335662365,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3519345298409462,
       "step": 28
     },
     {
-      "completion_length": 3597.824462890625,
+      "completion_length": 3638.3036499023438,
       "epoch": 0.8111888111888111,
-      "grad_norm": 0.2858404815196991,
-      "kl": 0.168212890625,
-      "learning_rate": 2.3180194846605367e-06,
-      "loss": 0.0067,
-      "reward": 0.6830357164144516,
-      "reward_std": 0.22093605995178223,
-      "rewards/accuracy_reward": 0.2946428619325161,
+      "grad_norm": 0.16842831671237946,
+      "kl": 0.206298828125,
+      "learning_rate": 9.334975706451863e-06,
+      "loss": 0.0083,
+      "reward": 0.6986607238650322,
+      "reward_std": 0.2177316304296255,
+      "rewards/accuracy_reward": 0.3035714365541935,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3883928582072258,
+      "rewards/tag_count_reward": 0.3950892984867096,
       "step": 29
     },
     {
-      "completion_length": 3639.58935546875,
+      "completion_length": 3669.9852294921875,
       "epoch": 0.8391608391608392,
-      "grad_norm": 0.18465252220630646,
-      "kl": 0.197021484375,
-      "learning_rate": 2.021452959867684e-06,
-      "loss": 0.0079,
-      "reward": 0.5840773954987526,
-      "reward_std": 0.2365283314138651,
-      "rewards/accuracy_reward": 0.20535714458674192,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3787202388048172,
+      "grad_norm": 0.3207129240036011,
+      "kl": 0.263671875,
+      "learning_rate": 9.256726238631721e-06,
+      "loss": 0.0105,
+      "reward": 0.5602678805589676,
+      "reward_std": 0.17984714545309544,
+      "rewards/accuracy_reward": 0.19345238246023655,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3668154776096344,
       "step": 30
     },
     {
-      "completion_length": 3431.7291870117188,
+      "completion_length": 3451.3572387695312,
       "epoch": 0.8671328671328671,
-      "grad_norm": 0.18510738015174866,
-      "kl": 0.185302734375,
-      "learning_rate": 1.7583867446385461e-06,
-      "loss": 0.0074,
-      "reward": 0.7075892835855484,
-      "reward_std": 0.22301654145121574,
-      "rewards/accuracy_reward": 0.3065476305782795,
+      "grad_norm": 0.2888355553150177,
+      "kl": 0.24609375,
+      "learning_rate": 9.174536483757449e-06,
+      "loss": 0.0099,
+      "reward": 0.6800595372915268,
+      "reward_std": 0.22582022473216057,
+      "rewards/accuracy_reward": 0.2767857201397419,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4010416716337204,
+      "rewards/tag_count_reward": 0.4032738208770752,
       "step": 31
     },
     {
-      "completion_length": 3274.8601684570312,
+      "completion_length": 3316.4137573242188,
       "epoch": 0.8951048951048951,
-      "grad_norm": 0.15279972553253174,
-      "kl": 0.181396484375,
-      "learning_rate": 1.531354310432403e-06,
-      "loss": 0.0073,
-      "reward": 0.6845238208770752,
-      "reward_std": 0.08698987402021885,
-      "rewards/accuracy_reward": 0.2886904813349247,
+      "grad_norm": 0.1782606989145279,
+      "kl": 0.2578125,
+      "learning_rate": 9.088492647532244e-06,
+      "loss": 0.0103,
+      "reward": 0.6897321492433548,
+      "reward_std": 0.14058080688118935,
+      "rewards/accuracy_reward": 0.2976190559566021,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3958333432674408,
+      "rewards/tag_count_reward": 0.3921131044626236,
       "step": 32
     },
     {
-      "completion_length": 3458.9524536132812,
+      "completion_length": 3457.824462890625,
       "epoch": 0.9230769230769231,
-      "grad_norm": 0.18014280498027802,
-      "kl": 0.208984375,
-      "learning_rate": 1.3425421036992098e-06,
-      "loss": 0.0083,
-      "reward": 0.7135416865348816,
-      "reward_std": 0.207016596570611,
-      "rewards/accuracy_reward": 0.3154761977493763,
+      "grad_norm": 0.20152917504310608,
+      "kl": 0.3046875,
+      "learning_rate": 8.998684978058423e-06,
+      "loss": 0.0122,
+      "reward": 0.7008928656578064,
+      "reward_std": 0.23007170855998993,
+      "rewards/accuracy_reward": 0.3035714328289032,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.398065485060215,
+      "rewards/tag_count_reward": 0.3973214402794838,
       "step": 33
     },
     {
-      "completion_length": 3083.3065795898438,
+      "completion_length": 3092.1190795898438,
       "epoch": 0.951048951048951,
-      "grad_norm": 0.5020937323570251,
-      "kl": 0.207275390625,
-      "learning_rate": 1.1937684892050606e-06,
-      "loss": 0.0083,
-      "reward": 0.7477678805589676,
-      "reward_std": 0.21528683602809906,
-      "rewards/accuracy_reward": 0.3214285746216774,
+      "grad_norm": 0.20092906057834625,
+      "kl": 0.3056640625,
+      "learning_rate": 8.905207671179629e-06,
+      "loss": 0.0122,
+      "reward": 0.7663690745830536,
+      "reward_std": 0.22053436189889908,
+      "rewards/accuracy_reward": 0.3333333358168602,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4263392984867096,
+      "rewards/tag_count_reward": 0.4330357238650322,
       "step": 34
     },
     {
-      "completion_length": 3171.6221313476562,
+      "completion_length": 3940.1637573242188,
       "epoch": 0.9790209790209791,
-      "grad_norm": 0.17938590049743652,
-      "kl": 0.200439453125,
-      "learning_rate": 1.0864662381854632e-06,
-      "loss": 0.008,
-      "reward": 0.721726194024086,
-      "reward_std": 0.15224073268473148,
-      "rewards/accuracy_reward": 0.31845238991081715,
+      "grad_norm": 0.1926574409008026,
+      "kl": 0.3583984375,
+      "learning_rate": 8.808158771682402e-06,
+      "loss": 0.0143,
+      "reward": 0.4360119104385376,
+      "reward_std": 0.14251151774078608,
+      "rewards/accuracy_reward": 0.11607143329456449,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4032738208770752,
+      "rewards/tag_count_reward": 0.319940485060215,
       "step": 35
     },
     {
-      "completion_length": 3707.666748046875,
+      "completion_length": 4359.97802734375,
       "epoch": 1.0,
-      "grad_norm": 0.17938590049743652,
-      "kl": 0.20670572916666666,
-      "learning_rate": 1.0216687299751146e-06,
-      "loss": 0.0062,
-      "reward": 0.460317462682724,
-      "reward_std": 0.09649834036827087,
-      "rewards/accuracy_reward": 0.11111111473292112,
-      "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3492063581943512,
+      "grad_norm": 0.1926574409008026,
+      "kl": 0.400390625,
+      "learning_rate": 8.707640070460733e-06,
+      "loss": 0.012,
+      "reward": 0.3730158706506093,
+      "reward_std": 0.13879583527644476,
+      "rewards/accuracy_reward": 0.07936508022248745,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.2936507960160573,
       "step": 36
     },
     {
-      "epoch": 1.0,
-      "step": 36,
+      "completion_length": 3477.6101684570312,
+      "epoch": 1.027972027972028,
+      "grad_norm": 0.21748770773410797,
+      "kl": 0.3212890625,
+      "learning_rate": 8.60375699775147e-06,
+      "loss": 0.0129,
+      "reward": 0.4784226194024086,
+      "reward_std": 0.12951527908444405,
+      "rewards/accuracy_reward": 0.09523809631355107,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3831845372915268,
+      "step": 37
+    },
+    {
+      "completion_length": 3679.9732666015625,
+      "epoch": 1.055944055944056,
+      "grad_norm": 0.16126495599746704,
+      "kl": 0.36572265625,
+      "learning_rate": 8.496618512552566e-06,
+      "loss": 0.0146,
+      "reward": 0.4895833432674408,
+      "reward_std": 0.15181067399680614,
+      "rewards/accuracy_reward": 0.11011904990300536,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3794642984867096,
+      "step": 38
+    },
+    {
+      "completion_length": 3395.2559814453125,
+      "epoch": 1.083916083916084,
+      "grad_norm": 0.31568872928619385,
+      "kl": 0.35546875,
+      "learning_rate": 8.38633698834013e-06,
+      "loss": 0.0142,
+      "reward": 0.563988097012043,
+      "reward_std": 0.16420039534568787,
+      "rewards/accuracy_reward": 0.13988095615059137,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4241071492433548,
+      "step": 39
+    },
+    {
+      "completion_length": 3251.02978515625,
+      "epoch": 1.1118881118881119,
+      "grad_norm": 0.28431281447410583,
+      "kl": 0.376953125,
+      "learning_rate": 8.273028095204174e-06,
+      "loss": 0.015,
+      "reward": 0.5565476268529892,
+      "reward_std": 0.11341512016952038,
+      "rewards/accuracy_reward": 0.1398809552192688,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.416666679084301,
+      "step": 40
+    },
+    {
+      "completion_length": 3567.4495239257812,
+      "epoch": 1.1398601398601398,
+      "grad_norm": 0.28001177310943604,
+      "kl": 2.77392578125,
+      "learning_rate": 8.156810678526652e-06,
+      "loss": 0.1111,
+      "reward": 0.5498512089252472,
+      "reward_std": 0.1434172596782446,
+      "rewards/accuracy_reward": 0.16071428917348385,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3891369104385376,
+      "step": 41
+    },
+    {
+      "completion_length": 3581.7559814453125,
+      "epoch": 1.167832167832168,
+      "grad_norm": 164.8837127685547,
+      "kl": 0.478515625,
+      "learning_rate": 8.037806634329079e-06,
+      "loss": 0.0191,
+      "reward": 0.501488097012043,
+      "reward_std": 0.196690171957016,
+      "rewards/accuracy_reward": 0.10416666930541396,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3973214328289032,
+      "step": 42
+    },
+    {
+      "completion_length": 3735.9702758789062,
+      "epoch": 1.1958041958041958,
+      "grad_norm": 0.45882004499435425,
+      "kl": 0.5,
+      "learning_rate": 7.916140781420428e-06,
+      "loss": 0.02,
+      "reward": 0.5550595223903656,
+      "reward_std": 0.20235663652420044,
+      "rewards/accuracy_reward": 0.16666666977107525,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3883928582072258,
+      "step": 43
+    },
+    {
+      "completion_length": 3458.1281127929688,
+      "epoch": 1.2237762237762237,
+      "grad_norm": 0.7228833436965942,
+      "kl": 0.5859375,
+      "learning_rate": 7.791940730479435e-06,
+      "loss": 0.0234,
+      "reward": 0.5483631044626236,
+      "reward_std": 0.20634656865149736,
+      "rewards/accuracy_reward": 0.12797619216144085,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4203869104385376,
+      "step": 44
+    },
+    {
+      "completion_length": 3490.0357666015625,
+      "epoch": 1.2517482517482517,
+      "grad_norm": 0.43375658988952637,
+      "kl": 0.744140625,
+      "learning_rate": 7.665336750208624e-06,
+      "loss": 0.0297,
+      "reward": 0.5811012014746666,
+      "reward_std": 0.21103684417903423,
+      "rewards/accuracy_reward": 0.18452381365932524,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3965773954987526,
+      "step": 45
+    },
+    {
+      "completion_length": 3487.4435424804688,
+      "epoch": 1.2797202797202798,
+      "grad_norm": 1.1383850574493408,
+      "kl": 0.9482421875,
+      "learning_rate": 7.536461630700426e-06,
+      "loss": 0.0379,
+      "reward": 0.5952381193637848,
+      "reward_std": 0.24980730190873146,
+      "rewards/accuracy_reward": 0.16071428917348385,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4345238134264946,
+      "step": 46
+    },
+    {
+      "completion_length": 3268.75,
+      "epoch": 1.3076923076923077,
+      "grad_norm": 1.0813361406326294,
+      "kl": 1.525390625,
+      "learning_rate": 7.4054505441587075e-06,
+      "loss": 0.061,
+      "reward": 0.6049107313156128,
+      "reward_std": 0.17533794045448303,
+      "rewards/accuracy_reward": 0.11607143003493547,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.488839291036129,
+      "step": 47
+    },
+    {
+      "completion_length": 2660.7113037109375,
+      "epoch": 1.3356643356643356,
+      "grad_norm": 9.25616455078125,
+      "kl": 1.716796875,
+      "learning_rate": 7.272440903121792e-06,
+      "loss": 0.0687,
+      "reward": 0.6532738357782364,
+      "reward_std": 0.17452342063188553,
+      "rewards/accuracy_reward": 0.13095238571986556,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.522321417927742,
+      "step": 48
+    },
+    {
+      "completion_length": 2249.235137939453,
+      "epoch": 1.3636363636363638,
+      "grad_norm": 2.210108995437622,
+      "kl": 581.240234375,
+      "learning_rate": 7.137572216335695e-06,
+      "loss": 23.2541,
+      "reward": 0.7261904925107956,
+      "reward_std": 0.21592015214264393,
+      "rewards/accuracy_reward": 0.2172619104385376,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.508928582072258,
+      "step": 49
+    },
+    {
+      "completion_length": 2210.3809814453125,
+      "epoch": 1.3916083916083917,
+      "grad_norm": 24608.181640625,
+      "kl": 1.755859375,
+      "learning_rate": 7.000985942428694e-06,
+      "loss": 0.0702,
+      "reward": 0.8735119253396988,
+      "reward_std": 0.29524174705147743,
+      "rewards/accuracy_reward": 0.3690476268529892,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.5044643059372902,
+      "step": 50
+    },
+    {
+      "completion_length": 2277.0952758789062,
+      "epoch": 1.4195804195804196,
+      "grad_norm": 1.4279229640960693,
+      "kl": 2.052734375,
+      "learning_rate": 6.862825341540779e-06,
+      "loss": 0.0821,
+      "reward": 0.7269345223903656,
+      "reward_std": 0.25008974969387054,
+      "rewards/accuracy_reward": 0.21130953170359135,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.515625,
+      "step": 51
+    },
+    {
+      "completion_length": 2418.7025146484375,
+      "epoch": 1.4475524475524475,
+      "grad_norm": 16.061054229736328,
+      "kl": 1.5625,
+      "learning_rate": 6.723235325063544e-06,
+      "loss": 0.0625,
+      "reward": 0.8162202537059784,
+      "reward_std": 0.2910061627626419,
+      "rewards/accuracy_reward": 0.3065476268529892,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.509672611951828,
+      "step": 52
+    },
+    {
+      "completion_length": 2389.8839721679688,
+      "epoch": 1.4755244755244754,
+      "grad_norm": 1.9534995555877686,
+      "kl": 1.697265625,
+      "learning_rate": 6.582362303648142e-06,
+      "loss": 0.0679,
+      "reward": 0.82738097012043,
+      "reward_std": 0.27743372321128845,
+      "rewards/accuracy_reward": 0.3244047611951828,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.5029762089252472,
+      "step": 53
+    },
+    {
+      "completion_length": 2719.7142944335938,
+      "epoch": 1.5034965034965035,
+      "grad_norm": 1.6721709966659546,
+      "kl": 1.46875,
+      "learning_rate": 6.440354033640739e-06,
+      "loss": 0.0588,
+      "reward": 0.7924107164144516,
+      "reward_std": 0.2318180948495865,
+      "rewards/accuracy_reward": 0.27976191882044077,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.5126488208770752,
+      "step": 54
+    },
+    {
+      "completion_length": 2739.5684814453125,
+      "epoch": 1.5314685314685315,
+      "grad_norm": 4.0345778465271,
+      "kl": 1.296875,
+      "learning_rate": 6.297359462106504e-06,
+      "loss": 0.0519,
+      "reward": 0.9494047909975052,
+      "reward_std": 0.33292248100042343,
+      "rewards/accuracy_reward": 0.4226190596818924,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.526785708963871,
+      "step": 55
+    },
+    {
+      "completion_length": 3403.4136962890625,
+      "epoch": 1.5594405594405596,
+      "grad_norm": 0.4623602032661438,
+      "kl": 1.232421875,
+      "learning_rate": 6.1535285706047075e-06,
+      "loss": 0.0493,
+      "reward": 0.7641369104385376,
+      "reward_std": 0.2736235670745373,
+      "rewards/accuracy_reward": 0.2767857164144516,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4873512014746666,
+      "step": 56
+    },
+    {
+      "completion_length": 3603.52392578125,
+      "epoch": 1.5874125874125875,
+      "grad_norm": 0.5890073776245117,
+      "kl": 1.009765625,
+      "learning_rate": 6.00901221787878e-06,
+      "loss": 0.0404,
+      "reward": 0.8504464477300644,
+      "reward_std": 0.328320749104023,
+      "rewards/accuracy_reward": 0.3898809552192688,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4605654925107956,
+      "step": 57
+    },
+    {
+      "completion_length": 3894.0327758789062,
+      "epoch": 1.6153846153846154,
+      "grad_norm": 1.1042261123657227,
+      "kl": 1.07421875,
+      "learning_rate": 5.863961981626321e-06,
+      "loss": 0.043,
+      "reward": 0.677827388048172,
+      "reward_std": 0.29943743720650673,
+      "rewards/accuracy_reward": 0.27678571082651615,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4010416716337204,
+      "step": 58
+    },
+    {
+      "completion_length": 3703.5418090820312,
+      "epoch": 1.6433566433566433,
+      "grad_norm": 0.8175861239433289,
+      "kl": 0.8017578125,
+      "learning_rate": 5.718529999515018e-06,
+      "loss": 0.032,
+      "reward": 0.7596726417541504,
+      "reward_std": 0.31634173914790154,
+      "rewards/accuracy_reward": 0.348214291036129,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4114583432674408,
+      "step": 59
+    },
+    {
+      "completion_length": 3456.7232666015625,
+      "epoch": 1.6713286713286712,
+      "grad_norm": 0.6864073872566223,
+      "kl": 0.88671875,
+      "learning_rate": 5.572868809611258e-06,
+      "loss": 0.0355,
+      "reward": 0.7306547909975052,
+      "reward_std": 0.3101131170988083,
+      "rewards/accuracy_reward": 0.3244047649204731,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4062500074505806,
+      "step": 60
+    },
+    {
+      "completion_length": 3348.4910888671875,
+      "epoch": 1.6993006993006992,
+      "grad_norm": 1.040555715560913,
+      "kl": 0.91796875,
+      "learning_rate": 5.427131190388743e-06,
+      "loss": 0.0367,
+      "reward": 0.8162202388048172,
+      "reward_std": 0.31024138629436493,
+      "rewards/accuracy_reward": 0.3720238134264946,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4441964402794838,
+      "step": 61
+    },
+    {
+      "completion_length": 3336.732177734375,
+      "epoch": 1.7272727272727273,
+      "grad_norm": 1.47223699092865,
+      "kl": 1.1650390625,
+      "learning_rate": 5.281470000484985e-06,
+      "loss": 0.0466,
+      "reward": 0.76488097012043,
+      "reward_std": 0.2751483619213104,
+      "rewards/accuracy_reward": 0.33630953542888165,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4285714328289032,
+      "step": 62
+    },
+    {
+      "completion_length": 2883.4136962890625,
+      "epoch": 1.7552447552447552,
+      "grad_norm": 12.537298202514648,
+      "kl": 0.833984375,
+      "learning_rate": 5.136038018373682e-06,
+      "loss": 0.0333,
+      "reward": 0.8363095223903656,
+      "reward_std": 0.20866946317255497,
+      "rewards/accuracy_reward": 0.3630952425301075,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4732142984867096,
+      "step": 63
+    },
+    {
+      "completion_length": 3351.5625610351562,
+      "epoch": 1.7832167832167833,
+      "grad_norm": 1.4591819047927856,
+      "kl": 38.779296875,
+      "learning_rate": 4.9909877821212215e-06,
+      "loss": 1.5543,
+      "reward": 0.6175595447421074,
+      "reward_std": 0.17579150572419167,
+      "rewards/accuracy_reward": 0.1666666679084301,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4508928656578064,
+      "step": 64
+    },
+    {
+      "completion_length": 3067.014892578125,
+      "epoch": 1.8111888111888113,
+      "grad_norm": 1026.2440185546875,
+      "kl": 1.087890625,
+      "learning_rate": 4.8464714293952956e-06,
+      "loss": 0.0436,
+      "reward": 0.8541666865348816,
+      "reward_std": 0.2841528169810772,
+      "rewards/accuracy_reward": 0.3779761977493763,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.476190485060215,
+      "step": 65
+    },
+    {
+      "completion_length": 2978.4346313476562,
+      "epoch": 1.8391608391608392,
+      "grad_norm": 0.6207138299942017,
+      "kl": 1.2548828125,
+      "learning_rate": 4.702640537893498e-06,
+      "loss": 0.0502,
+      "reward": 0.7068452537059784,
+      "reward_std": 0.21755698695778847,
+      "rewards/accuracy_reward": 0.22916667396202683,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4776785746216774,
+      "step": 66
+    },
+    {
+      "completion_length": 2767.7798461914062,
+      "epoch": 1.867132867132867,
+      "grad_norm": 1.02067232131958,
+      "kl": 1.326171875,
+      "learning_rate": 4.559645966359263e-06,
+      "loss": 0.0531,
+      "reward": 0.7477678656578064,
+      "reward_std": 0.24407314509153366,
+      "rewards/accuracy_reward": 0.2678571455180645,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4799107238650322,
+      "step": 67
+    },
+    {
+      "completion_length": 2605.0506591796875,
+      "epoch": 1.895104895104895,
+      "grad_norm": 1.0706675052642822,
+      "kl": 1.21484375,
+      "learning_rate": 4.417637696351861e-06,
+      "loss": 0.0485,
+      "reward": 0.7604166865348816,
+      "reward_std": 0.16445039585232735,
+      "rewards/accuracy_reward": 0.2767857201397419,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4836309626698494,
+      "step": 68
+    },
+    {
+      "completion_length": 2840.5625610351562,
+      "epoch": 1.9230769230769231,
+      "grad_norm": 2.3417489528656006,
+      "kl": 1.453125,
+      "learning_rate": 4.2767646749364574e-06,
+      "loss": 0.0582,
+      "reward": 0.7202381193637848,
+      "reward_std": 0.2862440012395382,
+      "rewards/accuracy_reward": 0.2589285746216774,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4613095298409462,
+      "step": 69
+    },
+    {
+      "completion_length": 2714.1785888671875,
+      "epoch": 1.951048951048951,
+      "grad_norm": 1.3657090663909912,
+      "kl": 1.0849609375,
+      "learning_rate": 4.137174658459223e-06,
+      "loss": 0.0434,
+      "reward": 0.8623512089252472,
+      "reward_std": 0.26481272652745247,
+      "rewards/accuracy_reward": 0.3630952388048172,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4992559552192688,
+      "step": 70
+    },
+    {
+      "completion_length": 3436.5238647460938,
+      "epoch": 1.9790209790209792,
+      "grad_norm": 1.1092079877853394,
+      "kl": 1.720703125,
+      "learning_rate": 3.999014057571309e-06,
+      "loss": 0.069,
+      "reward": 0.6227678656578064,
+      "reward_std": 0.25541481748223305,
+      "rewards/accuracy_reward": 0.18750000558793545,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4352678582072258,
+      "step": 71
+    },
+    {
+      "completion_length": 4095.6446940104165,
+      "epoch": 2.0,
+      "grad_norm": 1.8023747205734253,
+      "kl": 1.6927083333333333,
+      "learning_rate": 3.862427783664306e-06,
+      "loss": 0.0507,
+      "reward": 0.4900793731212616,
+      "reward_std": 0.1970900297164917,
+      "rewards/accuracy_reward": 0.09920635198553403,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3908730248610179,
+      "step": 72
+    },
+    {
+      "completion_length": 3306.2947387695312,
+      "epoch": 2.027972027972028,
+      "grad_norm": 1.4027706384658813,
+      "kl": 1.6396484375,
+      "learning_rate": 3.7275590968782092e-06,
+      "loss": 0.0656,
+      "reward": 0.5691964253783226,
+      "reward_std": 0.21935388445854187,
+      "rewards/accuracy_reward": 0.11309524020180106,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4561011865735054,
+      "step": 73
+    },
+    {
+      "completion_length": 3458.6339721679688,
+      "epoch": 2.055944055944056,
+      "grad_norm": 0.5813905596733093,
+      "kl": 1.58984375,
+      "learning_rate": 3.5945494558412964e-06,
+      "loss": 0.0637,
+      "reward": 0.5885416716337204,
+      "reward_std": 0.19907395914196968,
+      "rewards/accuracy_reward": 0.1428571450524032,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4456845298409462,
+      "step": 74
+    },
+    {
+      "completion_length": 3205.52685546875,
+      "epoch": 2.0839160839160837,
+      "grad_norm": 0.4965643286705017,
+      "kl": 1.177734375,
+      "learning_rate": 3.463538369299576e-06,
+      "loss": 0.0472,
+      "reward": 0.6235119104385376,
+      "reward_std": 0.23606937006115913,
+      "rewards/accuracy_reward": 0.14285714365541935,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4806547686457634,
+      "step": 75
+    },
+    {
+      "completion_length": 3080.65771484375,
+      "epoch": 2.111888111888112,
+      "grad_norm": 5.0962443351745605,
+      "kl": 1.013671875,
+      "learning_rate": 3.334663249791378e-06,
+      "loss": 0.0406,
+      "reward": 0.5773809552192688,
+      "reward_std": 0.19794748164713383,
+      "rewards/accuracy_reward": 0.1220238134264946,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4553571492433548,
+      "step": 76
+    },
+    {
+      "completion_length": 3481.1190795898438,
+      "epoch": 2.13986013986014,
+      "grad_norm": 477.3974914550781,
+      "kl": 9.28515625,
+      "learning_rate": 3.208059269520568e-06,
+      "loss": 0.3739,
+      "reward": 0.5937500149011612,
+      "reward_std": 0.18974663130939007,
+      "rewards/accuracy_reward": 0.1815476231276989,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.412202388048172,
+      "step": 77
+    },
+    {
+      "completion_length": 3660.8959350585938,
+      "epoch": 2.167832167832168,
+      "grad_norm": 1.8242417573928833,
+      "kl": 0.9853515625,
+      "learning_rate": 3.0838592185795733e-06,
+      "loss": 0.0394,
+      "reward": 0.537202388048172,
+      "reward_std": 0.24809184670448303,
+      "rewards/accuracy_reward": 0.12797619309276342,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.409226194024086,
+      "step": 78
+    },
+    {
+      "completion_length": 3652.916748046875,
+      "epoch": 2.195804195804196,
+      "grad_norm": 2.0212318897247314,
+      "kl": 1.2041015625,
+      "learning_rate": 2.962193365670921e-06,
+      "loss": 0.0482,
+      "reward": 0.5483631044626236,
+      "reward_std": 0.21292699500918388,
+      "rewards/accuracy_reward": 0.14583333488553762,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.402529776096344,
+      "step": 79
+    },
+    {
+      "completion_length": 3400.9554443359375,
+      "epoch": 2.2237762237762237,
+      "grad_norm": 2.087148427963257,
+      "kl": 1.166015625,
+      "learning_rate": 2.84318932147335e-06,
+      "loss": 0.0467,
+      "reward": 0.5111607238650322,
+      "reward_std": 0.19660818576812744,
+      "rewards/accuracy_reward": 0.08928571827709675,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4218750074505806,
+      "step": 80
+    },
+    {
+      "completion_length": 3498.4970703125,
+      "epoch": 2.2517482517482517,
+      "grad_norm": 1.5509693622589111,
+      "kl": 1.2734375,
+      "learning_rate": 2.726971904795827e-06,
+      "loss": 0.0509,
+      "reward": 0.5863095372915268,
+      "reward_std": 0.2618163302540779,
+      "rewards/accuracy_reward": 0.16964286286383867,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4166666716337204,
+      "step": 81
+    },
+    {
+      "completion_length": 3320.2977294921875,
+      "epoch": 2.2797202797202796,
+      "grad_norm": 1.4535397291183472,
+      "kl": 1.3671875,
+      "learning_rate": 2.6136630116598715e-06,
+      "loss": 0.0547,
+      "reward": 0.633928582072258,
+      "reward_std": 0.2568051181733608,
+      "rewards/accuracy_reward": 0.19345238618552685,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4404762014746666,
+      "step": 82
+    },
+    {
+      "completion_length": 3347.7708740234375,
+      "epoch": 2.3076923076923075,
+      "grad_norm": 1.1822701692581177,
+      "kl": 1.642578125,
+      "learning_rate": 2.503381487447436e-06,
+      "loss": 0.0657,
+      "reward": 0.603422611951828,
+      "reward_std": 0.18616832420229912,
+      "rewards/accuracy_reward": 0.16369047947227955,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4397321566939354,
+      "step": 83
+    },
+    {
+      "completion_length": 3295.8721313476562,
+      "epoch": 2.335664335664336,
+      "grad_norm": 0.6564051508903503,
+      "kl": 1.583984375,
+      "learning_rate": 2.396243002248531e-06,
+      "loss": 0.0634,
+      "reward": 0.5677083432674408,
+      "reward_std": 0.21577723138034344,
+      "rewards/accuracy_reward": 0.1220238097012043,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4456845372915268,
+      "step": 84
+    },
+    {
+      "completion_length": 3335.4048461914062,
+      "epoch": 2.3636363636363638,
+      "grad_norm": 0.6571016907691956,
+      "kl": 1.974609375,
+      "learning_rate": 2.29235992953927e-06,
+      "loss": 0.0789,
+      "reward": 0.6979166716337204,
+      "reward_std": 0.298696992918849,
+      "rewards/accuracy_reward": 0.279761902987957,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4181547686457634,
+      "step": 85
+    },
+    {
+      "completion_length": 3272.0983276367188,
+      "epoch": 2.3916083916083917,
+      "grad_norm": 0.7644421458244324,
+      "kl": 1.79296875,
+      "learning_rate": 2.1918412283175996e-06,
+      "loss": 0.0716,
+      "reward": 0.8958333432674408,
+      "reward_std": 0.3616176173090935,
+      "rewards/accuracy_reward": 0.4761904776096344,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4196428656578064,
+      "step": 86
+    },
+    {
+      "completion_length": 3789.7709350585938,
+      "epoch": 2.4195804195804196,
+      "grad_norm": 1.8745439052581787,
+      "kl": 2.90625,
+      "learning_rate": 2.0947923288203713e-06,
+      "loss": 0.1163,
+      "reward": 0.6034226417541504,
+      "reward_std": 0.2801675796508789,
+      "rewards/accuracy_reward": 0.23511904664337635,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.368303582072258,
+      "step": 87
+    },
+    {
+      "completion_length": 3644.3690795898438,
+      "epoch": 2.4475524475524475,
+      "grad_norm": 1.7979298830032349,
+      "kl": 1.87109375,
+      "learning_rate": 2.0013150219415796e-06,
+      "loss": 0.0749,
+      "reward": 0.7566964477300644,
+      "reward_std": 0.34903113171458244,
+      "rewards/accuracy_reward": 0.345238097012043,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4114583432674408,
+      "step": 88
+    },
+    {
+      "completion_length": 3629.369140625,
+      "epoch": 2.4755244755244754,
+      "grad_norm": 0.8993220329284668,
+      "kl": 2.6484375,
+      "learning_rate": 1.9115073524677572e-06,
+      "loss": 0.1058,
+      "reward": 0.6696428805589676,
+      "reward_std": 0.3975626900792122,
+      "rewards/accuracy_reward": 0.2827381007373333,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.386904776096344,
+      "step": 89
+    },
+    {
+      "completion_length": 3657.0298461914062,
+      "epoch": 2.5034965034965033,
+      "grad_norm": 2.3261046409606934,
+      "kl": 2.380859375,
+      "learning_rate": 1.8254635162425506e-06,
+      "loss": 0.0952,
+      "reward": 0.6808035746216774,
+      "reward_std": 0.27695968747138977,
+      "rewards/accuracy_reward": 0.27976190857589245,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4010416716337204,
+      "step": 90
+    },
+    {
+      "completion_length": 3351.8870239257812,
+      "epoch": 2.5314685314685317,
+      "grad_norm": 1.173328161239624,
+      "kl": 1.958984375,
+      "learning_rate": 1.743273761368281e-06,
+      "loss": 0.0784,
+      "reward": 0.8422619253396988,
+      "reward_std": 0.3281675949692726,
+      "rewards/accuracy_reward": 0.4255952462553978,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.416666679084301,
+      "step": 91
+    },
+    {
+      "completion_length": 3638.806640625,
+      "epoch": 2.5594405594405596,
+      "grad_norm": 0.6548011898994446,
+      "kl": 1.91015625,
+      "learning_rate": 1.665024293548139e-06,
+      "loss": 0.0763,
+      "reward": 0.6852678656578064,
+      "reward_std": 0.273727435618639,
+      "rewards/accuracy_reward": 0.2886904776096344,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3965773954987526,
+      "step": 92
+    },
+    {
+      "completion_length": 3484.2113647460938,
+      "epoch": 2.5874125874125875,
+      "grad_norm": 0.9616138935089111,
+      "kl": 1.80859375,
+      "learning_rate": 1.5907971856683201e-06,
+      "loss": 0.0722,
+      "reward": 0.7834821343421936,
+      "reward_std": 0.32932380586862564,
+      "rewards/accuracy_reward": 0.3779762014746666,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4055059626698494,
+      "step": 93
+    },
+    {
+      "completion_length": 3699.3363037109375,
+      "epoch": 2.6153846153846154,
+      "grad_norm": 0.8441236615180969,
+      "kl": 1.95703125,
+      "learning_rate": 1.5206702917148948e-06,
+      "loss": 0.0782,
+      "reward": 0.751488097012043,
+      "reward_std": 0.3020743727684021,
+      "rewards/accuracy_reward": 0.35416667722165585,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3973214402794838,
+      "step": 94
+    },
+    {
+      "completion_length": 3259.8780517578125,
+      "epoch": 2.6433566433566433,
+      "grad_norm": 0.8067697882652283,
+      "kl": 1.578125,
+      "learning_rate": 1.4547171651157216e-06,
+      "loss": 0.063,
+      "reward": 0.8683035969734192,
+      "reward_std": 0.3106134869158268,
+      "rewards/accuracy_reward": 0.4494047686457634,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4188988134264946,
+      "step": 95
+    },
+    {
+      "completion_length": 3130.3958740234375,
+      "epoch": 2.6713286713286712,
+      "grad_norm": 0.9448930025100708,
+      "kl": 2.51953125,
+      "learning_rate": 1.3930069815930699e-06,
+      "loss": 0.1008,
+      "reward": 0.845238134264946,
+      "reward_std": 0.3288194462656975,
+      "rewards/accuracy_reward": 0.413690485060215,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4315476194024086,
+      "step": 96
+    },
+    {
+      "completion_length": 2959.2113037109375,
+      "epoch": 2.699300699300699,
+      "grad_norm": 86.35860443115234,
+      "kl": 1.66796875,
+      "learning_rate": 1.3356044666078316e-06,
+      "loss": 0.0667,
+      "reward": 0.9196428656578064,
+      "reward_std": 0.31560714542865753,
+      "rewards/accuracy_reward": 0.4702381044626236,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.449404776096344,
+      "step": 97
+    },
+    {
+      "completion_length": 3174.482177734375,
+      "epoch": 2.7272727272727275,
+      "grad_norm": 1.08586847782135,
+      "kl": 2.1953125,
+      "learning_rate": 1.2825698274714542e-06,
+      "loss": 0.0879,
+      "reward": 0.7790178656578064,
+      "reward_std": 0.2925742566585541,
+      "rewards/accuracy_reward": 0.36011906154453754,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4188988134264946,
+      "step": 98
+    },
+    {
+      "completion_length": 2669.029815673828,
+      "epoch": 2.755244755244755,
+      "grad_norm": 1.5489476919174194,
+      "kl": 1.087890625,
+      "learning_rate": 1.2339586901967831e-06,
+      "loss": 0.0435,
+      "reward": 0.82738097012043,
+      "reward_std": 0.1871817335486412,
+      "rewards/accuracy_reward": 0.3750000037252903,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4523809552192688,
+      "step": 99
+    },
+    {
+      "completion_length": 3378.9107666015625,
+      "epoch": 2.7832167832167833,
+      "grad_norm": 2.6001334190368652,
+      "kl": 1.6171875,
+      "learning_rate": 1.1898220411540584e-06,
+      "loss": 0.0646,
+      "reward": 0.5625000074505806,
+      "reward_std": 0.17016959004104137,
+      "rewards/accuracy_reward": 0.15476190857589245,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4077381044626236,
+      "step": 100
+    },
+    {
+      "completion_length": 3032.5596313476562,
+      "epoch": 2.8111888111888113,
+      "grad_norm": 0.5906639695167542,
+      "kl": 1.033203125,
+      "learning_rate": 1.15020617359325e-06,
+      "loss": 0.0413,
+      "reward": 0.8377976268529892,
+      "reward_std": 0.27019889280200005,
+      "rewards/accuracy_reward": 0.3750000074505806,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4627976268529892,
+      "step": 101
+    },
+    {
+      "completion_length": 3194.2351684570312,
+      "epoch": 2.839160839160839,
+      "grad_norm": 2.6414685249328613,
+      "kl": 1.2294921875,
+      "learning_rate": 1.1151526390888332e-06,
+      "loss": 0.0492,
+      "reward": 0.6569940447807312,
+      "reward_std": 0.21223976835608482,
+      "rewards/accuracy_reward": 0.2321428603027016,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4248512089252472,
+      "step": 102
+    },
+    {
+      "completion_length": 3116.6101684570312,
+      "epoch": 2.867132867132867,
+      "grad_norm": 1.5984132289886475,
+      "kl": 1.490234375,
+      "learning_rate": 1.0846982039579245e-06,
+      "loss": 0.0596,
+      "reward": 0.7373512089252472,
+      "reward_std": 0.23233365640044212,
+      "rewards/accuracy_reward": 0.2886904813349247,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4486607238650322,
+      "step": 103
+    },
+    {
+      "completion_length": 2812.1041870117188,
+      "epoch": 2.895104895104895,
+      "grad_norm": 2.6789627075195312,
+      "kl": 1.0205078125,
+      "learning_rate": 1.0588748106974919e-06,
+      "loss": 0.0408,
+      "reward": 0.7633928805589676,
+      "reward_std": 0.1795758679509163,
+      "rewards/accuracy_reward": 0.3125000037252903,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.450892873108387,
+      "step": 104
+    },
+    {
+      "completion_length": 3047.0625,
+      "epoch": 2.9230769230769234,
+      "grad_norm": 2.337068796157837,
+      "kl": 1.31640625,
+      "learning_rate": 1.0377095444810873e-06,
+      "loss": 0.0526,
+      "reward": 0.7068452537059784,
+      "reward_std": 0.2549416534602642,
+      "rewards/accuracy_reward": 0.2678571417927742,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4389881044626236,
+      "step": 105
+    },
+    {
+      "completion_length": 2715.509033203125,
+      "epoch": 2.951048951048951,
+      "grad_norm": 1.1716344356536865,
+      "kl": 0.943359375,
+      "learning_rate": 1.0212246047502374e-06,
+      "loss": 0.0378,
+      "reward": 0.8489583432674408,
+      "reward_std": 0.2508317902684212,
+      "rewards/accuracy_reward": 0.3690476343035698,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.4799107238650322,
+      "step": 106
+    },
+    {
+      "completion_length": 3682.6786499023438,
+      "epoch": 2.979020979020979,
+      "grad_norm": 1.8174355030059814,
+      "kl": 1.59375,
+      "learning_rate": 1.0094372819302978e-06,
+      "loss": 0.0637,
+      "reward": 0.5141369104385376,
+      "reward_std": 0.207677461206913,
+      "rewards/accuracy_reward": 0.1398809556849301,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3742559552192688,
+      "step": 107
+    },
+    {
+      "completion_length": 4012.118408203125,
+      "epoch": 3.0,
+      "grad_norm": 1.1122511625289917,
+      "kl": 1.6484375,
+      "learning_rate": 1.002359939295183e-06,
+      "loss": 0.0495,
+      "reward": 0.4861111243565877,
+      "reward_std": 0.16074346005916595,
+      "rewards/accuracy_reward": 0.10714286069075267,
+      "rewards/format_reward": 0.0,
+      "rewards/tag_count_reward": 0.3789682586987813,
+      "step": 108
+    },
+    {
+      "epoch": 3.0,
+      "step": 108,
       "total_flos": 0.0,
-      "train_loss": 0.004764942314540437,
-      "train_runtime": 16049.8199,
-      "train_samples_per_second": 0.062,
+      "train_loss": 0.2708671883675033,
+      "train_runtime": 44954.8681,
+      "train_samples_per_second": 0.067,
       "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 1,
-  "max_steps": 36,
+  "max_steps": 108,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 3,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {