Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen3-1.7B
-datasets: Blancy/secondfiltered-math220k-difficulty_stratified_10k_filtered_only_medium_difficulty
 library_name: transformers
 model_name: Qwen3-1.7B-Open-R1-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen3-1.7B-Open-R1-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) on the [Blancy/secondfiltered-math220k-difficulty_stratified_10k_filtered_only_medium_difficulty](https://huggingface.co/datasets/Blancy/secondfiltered-math220k-difficulty_stratified_10k_filtered_only_medium_difficulty) dataset.
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/wcdxh0jk) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/iqlnk5h1) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.8157487940822636,
-    "train_runtime": 11133.5212,
+    "train_loss": 0.005675484693879993,
+    "train_runtime": 16137.0184,
     "train_samples": 1000,
-    "train_samples_per_second": 0.09,
-    "train_steps_per_second": 0.003
+    "train_samples_per_second": 0.062,
+    "train_steps_per_second": 0.002
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.8157487940822636,
-    "train_runtime": 11133.5212,
+    "train_loss": 0.005675484693879993,
+    "train_runtime": 16137.0184,
     "train_samples": 1000,
-    "train_samples_per_second": 0.09,
-    "train_steps_per_second": 0.003
+    "train_samples_per_second": 0.062,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -10,517 +10,517 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 3740.9702758789062,
+      "completion_length": 3512.2977294921875,
       "epoch": 0.027972027972027972,
-      "grad_norm": 2.5353879928588867,
+      "grad_norm": 0.8604350090026855,
       "kl": 0.0,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.5535714328289032,
-      "reward_std": 0.23917681723833084,
-      "rewards/accuracy_reward": 0.11011904943734407,
+      "reward": 0.4687500149011612,
+      "reward_std": 0.12072492484003305,
+      "rewards/accuracy_reward": 0.08333333441987634,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.443452388048172,
+      "rewards/tag_count_reward": 0.3854166716337204,
       "step": 1
     },
     {
-      "completion_length": 3610.02685546875,
+      "completion_length": 3691.6666870117188,
       "epoch": 0.055944055944055944,
-      "grad_norm": 1.1924161911010742,
+      "grad_norm": 0.3791728615760803,
       "kl": 0.0,
       "learning_rate": 2.5e-06,
       "loss": 0.0,
-      "reward": 0.6622024029493332,
-      "reward_std": 0.2192090731114149,
-      "rewards/accuracy_reward": 0.2113095261156559,
+      "reward": 0.4680059477686882,
+      "reward_std": 0.1798743773251772,
+      "rewards/accuracy_reward": 0.09226190531626344,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.450892873108387,
+      "rewards/tag_count_reward": 0.3757440596818924,
       "step": 2
     },
     {
-      "completion_length": 3514.4406127929688,
+      "completion_length": 3429.5089721679688,
       "epoch": 0.08391608391608392,
-      "grad_norm": 1.2518582344055176,
-      "kl": 0.001033782958984375,
+      "grad_norm": 0.5100764036178589,
+      "kl": 0.0008497238159179688,
       "learning_rate": 5e-06,
       "loss": 0.0,
-      "reward": 0.5863095298409462,
-      "reward_std": 0.2138436622917652,
-      "rewards/accuracy_reward": 0.13690476515330374,
+      "reward": 0.5654762014746666,
+      "reward_std": 0.14018218219280243,
+      "rewards/accuracy_reward": 0.1428571455180645,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4494047611951828,
+      "rewards/tag_count_reward": 0.4226190596818924,
       "step": 3
     },
     {
-      "completion_length": 3706.636962890625,
+      "completion_length": 3350.5059814453125,
       "epoch": 0.11188811188811189,
-      "grad_norm": 1.1967724561691284,
-      "kl": 0.004276275634765625,
+      "grad_norm": 0.5230619311332703,
+      "kl": 0.001438140869140625,
       "learning_rate": 7.500000000000001e-06,
-      "loss": 0.0002,
-      "reward": 0.6011904999613762,
-      "reward_std": 0.1717666843906045,
-      "rewards/accuracy_reward": 0.1726190522313118,
+      "loss": 0.0001,
+      "reward": 0.5230654776096344,
+      "reward_std": 0.1375708756968379,
+      "rewards/accuracy_reward": 0.11011904943734407,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4285714402794838,
+      "rewards/tag_count_reward": 0.4129464328289032,
       "step": 4
     },
     {
-      "completion_length": 3468.27978515625,
+      "completion_length": 3646.4851684570312,
       "epoch": 0.13986013986013987,
-      "grad_norm": 1.1130006313323975,
-      "kl": 0.02197265625,
+      "grad_norm": 0.8575068712234497,
+      "kl": 0.00823211669921875,
       "learning_rate": 1e-05,
-      "loss": 0.0009,
-      "reward": 0.6629464626312256,
-      "reward_std": 0.26501624286174774,
-      "rewards/accuracy_reward": 0.2380952388048172,
+      "loss": 0.0003,
+      "reward": 0.5424107164144516,
+      "reward_std": 0.20411022752523422,
+      "rewards/accuracy_reward": 0.12500000279396772,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4248512014746666,
+      "rewards/tag_count_reward": 0.4174107238650322,
       "step": 5
     },
     {
-      "completion_length": 3647.2382202148438,
+      "completion_length": 3599.0298461914062,
       "epoch": 0.16783216783216784,
-      "grad_norm": 0.6197645664215088,
-      "kl": 0.04852294921875,
+      "grad_norm": 38.98723602294922,
+      "kl": 0.415771484375,
       "learning_rate": 9.978331270024887e-06,
-      "loss": 0.0019,
-      "reward": 0.5558035895228386,
-      "reward_std": 0.19512668810784817,
-      "rewards/accuracy_reward": 0.1011904813349247,
+      "loss": 0.0166,
+      "reward": 0.5632440447807312,
+      "reward_std": 0.2680988386273384,
+      "rewards/accuracy_reward": 0.06547619216144085,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4546131044626236,
+      "rewards/tag_count_reward": 0.497767873108387,
       "step": 6
     },
     {
-      "completion_length": 3054.0327758789062,
+      "completion_length": 3781.4287109375,
       "epoch": 0.1958041958041958,
-      "grad_norm": 0.3058634102344513,
-      "kl": 0.1123046875,
+      "grad_norm": 0.22663027048110962,
+      "kl": 0.03509521484375,
       "learning_rate": 9.913533761814537e-06,
-      "loss": 0.0045,
-      "reward": 0.9293155074119568,
-      "reward_std": 0.32718629390001297,
-      "rewards/accuracy_reward": 0.2589285708963871,
+      "loss": 0.0014,
+      "reward": 0.529017873108387,
+      "reward_std": 0.16097233816981316,
+      "rewards/accuracy_reward": 0.1458333358168602,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.6703869253396988,
+      "rewards/tag_count_reward": 0.3831845298409462,
       "step": 7
     },
     {
-      "completion_length": 3840.2678833007812,
+      "completion_length": 3490.544677734375,
       "epoch": 0.22377622377622378,
-      "grad_norm": 0.20688477158546448,
-      "kl": 0.1109619140625,
+      "grad_norm": 0.6074256300926208,
+      "kl": 0.0587158203125,
       "learning_rate": 9.80623151079494e-06,
-      "loss": 0.0044,
-      "reward": 0.9114583432674408,
-      "reward_std": 0.45086005330085754,
-      "rewards/accuracy_reward": 0.4523809552192688,
+      "loss": 0.0023,
+      "reward": 0.5052083507180214,
+      "reward_std": 0.21143596433103085,
+      "rewards/accuracy_reward": 0.10416666883975267,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.459077388048172,
+      "rewards/tag_count_reward": 0.4010416716337204,
       "step": 8
     },
     {
-      "completion_length": 3944.8988647460938,
+      "completion_length": 3470.9791870117188,
       "epoch": 0.2517482517482518,
-      "grad_norm": 0.22741489112377167,
-      "kl": 0.127685546875,
+      "grad_norm": 0.22321581840515137,
+      "kl": 0.058837890625,
       "learning_rate": 9.65745789630079e-06,
-      "loss": 0.0051,
-      "reward": 0.7061012089252472,
-      "reward_std": 0.36991604790091515,
-      "rewards/accuracy_reward": 0.3005952350795269,
+      "loss": 0.0024,
+      "reward": 0.6011904776096344,
+      "reward_std": 0.23324432224035263,
+      "rewards/accuracy_reward": 0.19940476678311825,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4055059552192688,
+      "rewards/tag_count_reward": 0.4017857238650322,
       "step": 9
     },
     {
-      "completion_length": 3999.8125610351562,
+      "completion_length": 3337.8363647460938,
       "epoch": 0.27972027972027974,
-      "grad_norm": 0.15535762906074524,
-      "kl": 0.155029296875,
+      "grad_norm": 0.23314569890499115,
+      "kl": 0.059814453125,
       "learning_rate": 9.468645689567599e-06,
-      "loss": 0.0062,
-      "reward": 0.836309552192688,
-      "reward_std": 0.403655294328928,
-      "rewards/accuracy_reward": 0.4345238208770752,
+      "loss": 0.0024,
+      "reward": 0.6495535969734192,
+      "reward_std": 0.19363375008106232,
+      "rewards/accuracy_reward": 0.2291666679084301,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4017857164144516,
+      "rewards/tag_count_reward": 0.4203869178891182,
       "step": 10
     },
     {
-      "completion_length": 4248.943420410156,
+      "completion_length": 3455.556640625,
       "epoch": 0.3076923076923077,
-      "grad_norm": 0.16839423775672913,
-      "kl": 0.216064453125,
+      "grad_norm": 0.16831204295158386,
+      "kl": 0.06622314453125,
       "learning_rate": 9.241613255361455e-06,
-      "loss": 0.0087,
-      "reward": 0.5141369104385376,
-      "reward_std": 0.3293060325086117,
-      "rewards/accuracy_reward": 0.19940476678311825,
+      "loss": 0.0027,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.12359252013266087,
+      "rewards/accuracy_reward": 0.13392857275903225,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3147321417927742,
+      "rewards/tag_count_reward": 0.3869047686457634,
       "step": 11
     },
     {
-      "completion_length": 4181.681579589844,
+      "completion_length": 3545.419677734375,
       "epoch": 0.3356643356643357,
-      "grad_norm": 0.21270643174648285,
-      "kl": 0.28564453125,
+      "grad_norm": 0.20184171199798584,
+      "kl": 0.07110595703125,
       "learning_rate": 8.978547040132317e-06,
-      "loss": 0.0114,
-      "reward": 0.560267873108387,
-      "reward_std": 0.32143769413232803,
-      "rewards/accuracy_reward": 0.22619048319756985,
+      "loss": 0.0028,
+      "reward": 0.4813988134264946,
+      "reward_std": 0.15685281716287136,
+      "rewards/accuracy_reward": 0.10416667070239782,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.334077388048172,
+      "rewards/tag_count_reward": 0.3772321492433548,
       "step": 12
     },
     {
-      "completion_length": 4254.375,
+      "completion_length": 3602.4286499023438,
       "epoch": 0.36363636363636365,
-      "grad_norm": 0.20891696214675903,
-      "kl": 0.341796875,
+      "grad_norm": 0.6997456550598145,
+      "kl": 0.0869140625,
       "learning_rate": 8.681980515339464e-06,
-      "loss": 0.0137,
-      "reward": 0.5074404925107956,
-      "reward_std": 0.3785836473107338,
-      "rewards/accuracy_reward": 0.1994047649204731,
+      "loss": 0.0035,
+      "reward": 0.6718750074505806,
+      "reward_std": 0.1674486780539155,
+      "rewards/accuracy_reward": 0.3005952499806881,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3080357238650322,
+      "rewards/tag_count_reward": 0.3712797686457634,
       "step": 13
     },
     {
-      "completion_length": 2929.9107666015625,
+      "completion_length": 3812.681640625,
       "epoch": 0.3916083916083916,
-      "grad_norm": 7.643617153167725,
-      "kl": 0.830078125,
+      "grad_norm": 0.19167593121528625,
+      "kl": 0.0965576171875,
       "learning_rate": 8.354769778736407e-06,
-      "loss": 0.0332,
-      "reward": 0.758928582072258,
-      "reward_std": 0.3943299725651741,
-      "rewards/accuracy_reward": 0.2708333395421505,
+      "loss": 0.0039,
+      "reward": 0.693452388048172,
+      "reward_std": 0.24091331334784627,
+      "rewards/accuracy_reward": 0.3392857201397419,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.4880952462553978,
+      "rewards/tag_count_reward": 0.3541666716337204,
       "step": 14
     },
     {
-      "completion_length": 1188.7470397949219,
+      "completion_length": 4161.53271484375,
       "epoch": 0.4195804195804196,
-      "grad_norm": 28577.451171875,
-      "kl": 512.5,
+      "grad_norm": 0.22708754241466522,
+      "kl": 0.108642578125,
       "learning_rate": 8.00006604858821e-06,
-      "loss": 20.5117,
-      "reward": 0.9099702686071396,
-      "reward_std": 0.3528708219528198,
-      "rewards/accuracy_reward": 0.23214286379516125,
+      "loss": 0.0043,
+      "reward": 0.4962797611951828,
+      "reward_std": 0.17475327849388123,
+      "rewards/accuracy_reward": 0.190476194024086,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.677827388048172,
+      "rewards/tag_count_reward": 0.305803582072258,
       "step": 15
     },
     {
-      "completion_length": 1025.5803833007812,
+      "completion_length": 4106.3929443359375,
       "epoch": 0.44755244755244755,
-      "grad_norm": 7.379767894744873,
-      "kl": 2.345703125,
+      "grad_norm": 0.20696592330932617,
+      "kl": 0.120361328125,
       "learning_rate": 7.621285315716991e-06,
-      "loss": 0.0937,
-      "reward": 0.8757440596818924,
-      "reward_std": 0.3126339688897133,
-      "rewards/accuracy_reward": 0.16666666883975267,
+      "loss": 0.0048,
+      "reward": 0.538690485060215,
+      "reward_std": 0.1706959567964077,
+      "rewards/accuracy_reward": 0.22023809887468815,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.709077388048172,
+      "rewards/tag_count_reward": 0.318452388048172,
       "step": 16
     },
     {
-      "completion_length": 1385.9315795898438,
+      "completion_length": 4136.211486816406,
       "epoch": 0.4755244755244755,
-      "grad_norm": 1.5446727275848389,
-      "kl": 0.7958984375,
+      "grad_norm": 0.2171836495399475,
+      "kl": 0.145751953125,
       "learning_rate": 7.222075445642904e-06,
-      "loss": 0.0318,
-      "reward": 0.8377976417541504,
-      "reward_std": 0.4007321819663048,
-      "rewards/accuracy_reward": 0.2648809626698494,
+      "loss": 0.0058,
+      "reward": 0.5014881044626236,
+      "reward_std": 0.15721454098820686,
+      "rewards/accuracy_reward": 0.19642857275903225,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.5729166641831398,
+      "rewards/tag_count_reward": 0.3050595223903656,
       "step": 17
     },
     {
-      "completion_length": 1099.3571472167969,
+      "completion_length": 4130.928527832031,
       "epoch": 0.5034965034965035,
-      "grad_norm": 2.3834547996520996,
-      "kl": 1.29296875,
+      "grad_norm": 0.12546850740909576,
+      "kl": 0.1390380859375,
       "learning_rate": 6.80628104764508e-06,
-      "loss": 0.0518,
-      "reward": 0.7760416716337204,
-      "reward_std": 0.37692204862833023,
-      "rewards/accuracy_reward": 0.16666666977107525,
+      "loss": 0.0056,
+      "reward": 0.4866071492433548,
+      "reward_std": 0.14047462213784456,
+      "rewards/accuracy_reward": 0.18154762126505375,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.6093750298023224,
+      "rewards/tag_count_reward": 0.3050595298409462,
       "step": 18
     },
     {
-      "completion_length": 705.9643096923828,
+      "completion_length": 3794.3424072265625,
       "epoch": 0.5314685314685315,
-      "grad_norm": 1.268905758857727,
-      "kl": 2.6015625,
+      "grad_norm": 0.1713448166847229,
+      "kl": 0.15673828125,
       "learning_rate": 6.377906449072578e-06,
-      "loss": 0.104,
-      "reward": 1.0922618806362152,
-      "reward_std": 0.2668025754392147,
-      "rewards/accuracy_reward": 0.19940476678311825,
+      "loss": 0.0063,
+      "reward": 0.6927083432674408,
+      "reward_std": 0.22955949790775776,
+      "rewards/accuracy_reward": 0.3482142947614193,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.8928571492433548,
+      "rewards/tag_count_reward": 0.3444940596818924,
       "step": 19
     },
     {
-      "completion_length": 626.8452758789062,
+      "completion_length": 4126.431640625,
       "epoch": 0.5594405594405595,
-      "grad_norm": 1.6185439825057983,
-      "kl": 2.73828125,
+      "grad_norm": 0.13459710776805878,
+      "kl": 0.175537109375,
       "learning_rate": 5.9410771314830255e-06,
-      "loss": 0.1096,
-      "reward": 1.064732164144516,
-      "reward_std": 0.2738974019885063,
-      "rewards/accuracy_reward": 0.1607142873108387,
+      "loss": 0.007,
+      "reward": 0.4598214402794838,
+      "reward_std": 0.13734917528927326,
+      "rewards/accuracy_reward": 0.16369048226624727,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9040178805589676,
+      "rewards/tag_count_reward": 0.2961309477686882,
       "step": 20
     },
     {
-      "completion_length": 654.1041717529297,
+      "completion_length": 4043.2351684570312,
       "epoch": 0.5874125874125874,
-      "grad_norm": 0.9367347359657288,
-      "kl": 2.76171875,
+      "grad_norm": 0.28960075974464417,
+      "kl": 0.16748046875,
       "learning_rate": 5.500000000000001e-06,
-      "loss": 0.1106,
-      "reward": 0.9836309850215912,
-      "reward_std": 0.2889478802680969,
-      "rewards/accuracy_reward": 0.08630952518433332,
+      "loss": 0.0067,
+      "reward": 0.5342261865735054,
+      "reward_std": 0.22838787361979485,
+      "rewards/accuracy_reward": 0.21428572200238705,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.8973214328289032,
+      "rewards/tag_count_reward": 0.319940485060215,
       "step": 21
     },
     {
-      "completion_length": 613.5803680419922,
+      "completion_length": 4076.5684814453125,
       "epoch": 0.6153846153846154,
-      "grad_norm": 2.1199615001678467,
-      "kl": 2.8359375,
+      "grad_norm": 0.15393978357315063,
+      "kl": 0.172607421875,
       "learning_rate": 5.0589228685169776e-06,
-      "loss": 0.1135,
-      "reward": 1.02976194024086,
-      "reward_std": 0.263757660984993,
-      "rewards/accuracy_reward": 0.1220238134264946,
+      "loss": 0.0069,
+      "reward": 0.5535714477300644,
+      "reward_std": 0.1490439474582672,
+      "rewards/accuracy_reward": 0.2410714328289032,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9077381044626236,
+      "rewards/tag_count_reward": 0.3125000074505806,
       "step": 22
     },
     {
-      "completion_length": 627.8363342285156,
+      "completion_length": 3821.038818359375,
       "epoch": 0.6433566433566433,
-      "grad_norm": 0.6018344163894653,
-      "kl": 2.61328125,
+      "grad_norm": 0.21775129437446594,
+      "kl": 0.189697265625,
       "learning_rate": 4.622093550927423e-06,
-      "loss": 0.1043,
-      "reward": 1.0096726268529892,
-      "reward_std": 0.23419346287846565,
-      "rewards/accuracy_reward": 0.10416666883975267,
+      "loss": 0.0076,
+      "reward": 0.7075892984867096,
+      "reward_std": 0.2531384788453579,
+      "rewards/accuracy_reward": 0.3630952499806881,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.90550597012043,
+      "rewards/tag_count_reward": 0.3444940596818924,
       "step": 23
     },
     {
-      "completion_length": 550.7351455688477,
+      "completion_length": 3507.6101684570312,
       "epoch": 0.6713286713286714,
-      "grad_norm": 1.0858573913574219,
-      "kl": 2.90234375,
+      "grad_norm": 0.14808182418346405,
+      "kl": 0.18017578125,
       "learning_rate": 4.193718952354921e-06,
-      "loss": 0.116,
-      "reward": 1.1629464626312256,
-      "reward_std": 0.2896396704018116,
-      "rewards/accuracy_reward": 0.2321428619325161,
+      "loss": 0.0072,
+      "reward": 0.7343750149011612,
+      "reward_std": 0.18677468597888947,
+      "rewards/accuracy_reward": 0.3690476231276989,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.930803582072258,
+      "rewards/tag_count_reward": 0.3653273805975914,
       "step": 24
     },
     {
-      "completion_length": 529.0982208251953,
+      "completion_length": 3405.7262573242188,
       "epoch": 0.6993006993006993,
-      "grad_norm": 1.6778327226638794,
-      "kl": 3.05859375,
+      "grad_norm": 0.1778838038444519,
+      "kl": 0.1806640625,
       "learning_rate": 3.777924554357096e-06,
-      "loss": 0.1223,
-      "reward": 1.1614583432674408,
-      "reward_std": 0.28430329635739326,
-      "rewards/accuracy_reward": 0.2291666679084301,
+      "loss": 0.0072,
+      "reward": 0.8072916865348816,
+      "reward_std": 0.28540152311325073,
+      "rewards/accuracy_reward": 0.4196428582072258,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9322916716337204,
+      "rewards/tag_count_reward": 0.3876488208770752,
       "step": 25
     },
     {
-      "completion_length": 686.3035736083984,
+      "completion_length": 3539.7589721679688,
       "epoch": 0.7272727272727273,
-      "grad_norm": 1.1516579389572144,
-      "kl": 1.87890625,
+      "grad_norm": 10.079139709472656,
+      "kl": 0.380615234375,
       "learning_rate": 3.378714684283011e-06,
-      "loss": 0.075,
-      "reward": 1.0297619253396988,
-      "reward_std": 0.27740226313471794,
-      "rewards/accuracy_reward": 0.1279761923942715,
+      "loss": 0.0152,
+      "reward": 0.7224702537059784,
+      "reward_std": 0.17052935622632504,
+      "rewards/accuracy_reward": 0.3690476268529892,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9017857313156128,
+      "rewards/tag_count_reward": 0.3534226268529892,
       "step": 26
     },
     {
-      "completion_length": 625.6250305175781,
+      "completion_length": 3090.5238647460938,
       "epoch": 0.7552447552447552,
-      "grad_norm": 1.0151987075805664,
-      "kl": 2.1953125,
+      "grad_norm": 0.18040025234222412,
+      "kl": 0.198486328125,
       "learning_rate": 2.9999339514117913e-06,
-      "loss": 0.0878,
-      "reward": 0.995535746216774,
-      "reward_std": 0.2647087723016739,
-      "rewards/accuracy_reward": 0.0922619067132473,
+      "loss": 0.0079,
+      "reward": 0.8050595223903656,
+      "reward_std": 0.17971179634332657,
+      "rewards/accuracy_reward": 0.3928571529686451,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9032738208770752,
+      "rewards/tag_count_reward": 0.412202388048172,
       "step": 27
     },
     {
-      "completion_length": 743.3601226806641,
+      "completion_length": 3675.3035888671875,
       "epoch": 0.7832167832167832,
-      "grad_norm": 1.265594720840454,
-      "kl": 2.052734375,
+      "grad_norm": 0.2067694365978241,
+      "kl": 0.204345703125,
       "learning_rate": 2.645230221263596e-06,
-      "loss": 0.0821,
-      "reward": 0.9337797909975052,
-      "reward_std": 0.2643532156944275,
-      "rewards/accuracy_reward": 0.041666666977107525,
+      "loss": 0.0082,
+      "reward": 0.5119047686457634,
+      "reward_std": 0.16108463145792484,
+      "rewards/accuracy_reward": 0.14285714365541935,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.8921131044626236,
+      "rewards/tag_count_reward": 0.3690476268529892,
       "step": 28
     },
     {
-      "completion_length": 704.5833587646484,
+      "completion_length": 3554.83935546875,
       "epoch": 0.8111888111888111,
-      "grad_norm": 1.292297601699829,
-      "kl": 1.8984375,
+      "grad_norm": 0.1498899906873703,
+      "kl": 0.177734375,
       "learning_rate": 2.3180194846605367e-06,
-      "loss": 0.0758,
-      "reward": 0.9635417014360428,
-      "reward_std": 0.2751614972949028,
-      "rewards/accuracy_reward": 0.06845238362438977,
+      "loss": 0.0071,
+      "reward": 0.7105654925107956,
+      "reward_std": 0.21086332947015762,
+      "rewards/accuracy_reward": 0.3035714291036129,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.8950892835855484,
+      "rewards/tag_count_reward": 0.4069940522313118,
       "step": 29
     },
     {
-      "completion_length": 726.8244323730469,
+      "completion_length": 3624.2232055664062,
       "epoch": 0.8391608391608392,
-      "grad_norm": 1.4452919960021973,
-      "kl": 1.96875,
+      "grad_norm": 0.19065174460411072,
+      "kl": 0.209716796875,
       "learning_rate": 2.021452959867684e-06,
-      "loss": 0.0786,
-      "reward": 0.9531250149011612,
-      "reward_std": 0.27541816234588623,
-      "rewards/accuracy_reward": 0.05357143096625805,
+      "loss": 0.0084,
+      "reward": 0.5654762014746666,
+      "reward_std": 0.18950912356376648,
+      "rewards/accuracy_reward": 0.18452381528913975,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.899553582072258,
+      "rewards/tag_count_reward": 0.3809523805975914,
       "step": 30
     },
     {
-      "completion_length": 729.4553527832031,
+      "completion_length": 3379.0714721679688,
       "epoch": 0.8671328671328671,
-      "grad_norm": 1.028757929801941,
-      "kl": 2.0859375,
+      "grad_norm": 0.19460894167423248,
+      "kl": 0.196044921875,
       "learning_rate": 1.7583867446385461e-06,
-      "loss": 0.0835,
-      "reward": 0.9769345372915268,
-      "reward_std": 0.2937622144818306,
-      "rewards/accuracy_reward": 0.08333333488553762,
+      "loss": 0.0079,
+      "reward": 0.7239583432674408,
+      "reward_std": 0.23103895224630833,
+      "rewards/accuracy_reward": 0.3125000074505806,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.8936012089252472,
+      "rewards/tag_count_reward": 0.4114583358168602,
       "step": 31
     },
     {
-      "completion_length": 755.1101379394531,
+      "completion_length": 3222.9405517578125,
       "epoch": 0.8951048951048951,
-      "grad_norm": 1.4110631942749023,
-      "kl": 2.0,
+      "grad_norm": 0.19526593387126923,
+      "kl": 0.1943359375,
       "learning_rate": 1.531354310432403e-06,
-      "loss": 0.08,
-      "reward": 0.9494047909975052,
-      "reward_std": 0.2776322588324547,
-      "rewards/accuracy_reward": 0.05357142956927419,
+      "loss": 0.0078,
+      "reward": 0.6793154776096344,
+      "reward_std": 0.13999886438250542,
+      "rewards/accuracy_reward": 0.279761902987957,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.8958333432674408,
+      "rewards/tag_count_reward": 0.3995535746216774,
       "step": 32
     },
     {
-      "completion_length": 725.3601226806641,
+      "completion_length": 3435.9376220703125,
       "epoch": 0.9230769230769231,
-      "grad_norm": 14314.6650390625,
-      "kl": 174.45703125,
+      "grad_norm": 0.16224603354930878,
+      "kl": 0.225341796875,
       "learning_rate": 1.3425421036992098e-06,
-      "loss": 6.9931,
-      "reward": 0.9568452686071396,
-      "reward_std": 0.27130044624209404,
-      "rewards/accuracy_reward": 0.059523812029510736,
+      "loss": 0.009,
+      "reward": 0.7209821492433548,
+      "reward_std": 0.18096437118947506,
+      "rewards/accuracy_reward": 0.3244047649204731,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.8973214477300644,
+      "rewards/tag_count_reward": 0.396577388048172,
       "step": 33
     },
     {
-      "completion_length": 645.8035736083984,
+      "completion_length": 2996.5952758789062,
       "epoch": 0.951048951048951,
-      "grad_norm": 4.487875938415527,
-      "kl": 2.46484375,
+      "grad_norm": 0.20968052744865417,
+      "kl": 0.20849609375,
       "learning_rate": 1.1937684892050606e-06,
-      "loss": 0.0984,
-      "reward": 0.9687500149011612,
-      "reward_std": 0.23901110514998436,
-      "rewards/accuracy_reward": 0.06845238292589784,
+      "loss": 0.0083,
+      "reward": 0.8229166865348816,
+      "reward_std": 0.2238299883902073,
+      "rewards/accuracy_reward": 0.380952388048172,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9002976417541504,
+      "rewards/tag_count_reward": 0.4419642984867096,
       "step": 34
     },
     {
-      "completion_length": 708.7559509277344,
+      "completion_length": 3923.5328979492188,
       "epoch": 0.9790209790209791,
-      "grad_norm": 1.2720866203308105,
-      "kl": 2.10546875,
+      "grad_norm": 0.2057952880859375,
+      "kl": 0.232666015625,
       "learning_rate": 1.0864662381854632e-06,
-      "loss": 0.0842,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.28124209865927696,
-      "rewards/accuracy_reward": 0.06547619262710214,
+      "loss": 0.0093,
+      "reward": 0.4434523954987526,
+      "reward_std": 0.1376960724592209,
+      "rewards/accuracy_reward": 0.11904762173071504,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9032738208770752,
+      "rewards/tag_count_reward": 0.3244047686457634,
       "step": 35
     },
     {
-      "completion_length": 167.14473724365234,
+      "completion_length": 4366.71484375,
       "epoch": 1.0,
-      "grad_norm": 1.2720866203308105,
-      "kl": 2.2916666666666665,
+      "grad_norm": 0.2057952880859375,
+      "kl": 0.24381510416666666,
       "learning_rate": 1.0216687299751146e-06,
-      "loss": 0.0688,
-      "reward": 0.9960317611694336,
-      "reward_std": 0.026725949719548225,
-      "rewards/accuracy_reward": 0.003968254042168458,
+      "loss": 0.0073,
+      "reward": 0.3928571442763011,
+      "reward_std": 0.11071915179491043,
+      "rewards/accuracy_reward": 0.09920635198553403,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.9920635024706522,
+      "rewards/tag_count_reward": 0.2936507960160573,
       "step": 36
     },
     {
       "epoch": 1.0,
       "step": 36,
       "total_flos": 0.0,
-      "train_loss": 0.8157487940822636,
-      "train_runtime": 11133.5212,
-      "train_samples_per_second": 0.09,
-      "train_steps_per_second": 0.003
+      "train_loss": 0.005675484693879993,
+      "train_runtime": 16137.0184,
+      "train_samples_per_second": 0.062,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 1,