Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen2.5-Math-7B
-datasets: Blancy/secondfiltered-math220k-difficulty_stratified_8k
 library_name: transformers
 model_name: Qwen-2.5-7B-Simple-RL
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen-2.5-7B-Simple-RL
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B) on the [Blancy/secondfiltered-math220k-difficulty_stratified_8k](https://huggingface.co/datasets/Blancy/secondfiltered-math220k-difficulty_stratified_8k) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/hp2j65o4) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/3y76zskm) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0010395098450702472,
-    "train_runtime": 14559.9882,
-    "train_samples": 7500,
-    "train_samples_per_second": 0.515,
+    "train_loss": 0.0006319970209619013,
+    "train_runtime": 19204.1352,
+    "train_samples": 10000,
+    "train_samples_per_second": 0.521,
     "train_steps_per_second": 0.004
 }

config.json

@@ -23,7 +23,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.49.0",
-  "use_cache": true,
+  "use_cache": false,
   "use_mrope": false,
   "use_sliding_window": false,
   "vocab_size": 152064

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a1eb6e5679e4d58dd28e87c12fa85b41e6f8649886af1b00e5e560fe9d132ea
+oid sha256:aae161abbf29198cd3b2b48df0e92aff65899add1650eeb5c8261fda9ba83786
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95cd0d8bf08fa8ef084bd405054d1ec4dfa7a3e1b856c6f80bcbdb25a01118a8
+oid sha256:0ced947bddeba4477ce8a9505cc27cf7c5032a3c7f427145b4d038f68294741e
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00f846aafa78bf493ca9d4cb53df3c719240a90c7046ca60255a7bf169ba1165
+oid sha256:f3daf3c87a42981f743aa78e8696d12f363f0f22d7ac8f722767b14bc04d4267
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:621840d94a3e10c3fd2b1d1227690bb2b13530236ad68b97eabacb16ccb7cd2c
+oid sha256:989a70e4f95e48a2b61859b8addca060fd1890cdf613172d4a3ce1820ad50e90
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0010395098450702472,
-    "train_runtime": 14559.9882,
-    "train_samples": 7500,
-    "train_samples_per_second": 0.515,
+    "train_loss": 0.0006319970209619013,
+    "train_runtime": 19204.1352,
+    "train_samples": 10000,
+    "train_samples_per_second": 0.521,
     "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -1,187 +1,239 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9893390191897654,
+  "epoch": 0.9984,
   "eval_steps": 100,
-  "global_step": 58,
+  "global_step": 78,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 743.5234680175781,
-      "epoch": 0.017057569296375266,
-      "grad_norm": 4.396385192871094,
+      "completion_length": 746.608283996582,
+      "epoch": 0.0128,
+      "grad_norm": 0.5155316591262817,
       "kl": 0.0,
-      "learning_rate": 5e-07,
+      "learning_rate": 3.75e-07,
       "loss": 0.0,
-      "reward": 0.026785715366713703,
-      "reward_std": 0.04944828595034778,
-      "rewards/accuracy_reward": 0.026785715366713703,
+      "reward": 0.14843750838190317,
+      "reward_std": 0.16338601242750883,
+      "rewards/accuracy_reward": 0.14843750838190317,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
-      "completion_length": 735.4506530761719,
-      "epoch": 0.08528784648187633,
-      "grad_norm": 0.44695502519607544,
-      "kl": 0.0003033876419067383,
-      "learning_rate": 2.5e-06,
+      "completion_length": 727.0990829467773,
+      "epoch": 0.064,
+      "grad_norm": 1.084052324295044,
+      "kl": 0.0003186464309692383,
+      "learning_rate": 1.875e-06,
       "loss": 0.0,
-      "reward": 0.032924108527367935,
-      "reward_std": 0.05200784042244777,
-      "rewards/accuracy_reward": 0.032924108527367935,
+      "reward": 0.1143973259604536,
+      "reward_std": 0.12805407610721886,
+      "rewards/accuracy_reward": 0.1143973259604536,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
-      "completion_length": 745.6826187133789,
-      "epoch": 0.17057569296375266,
-      "grad_norm": 0.509334921836853,
-      "kl": 0.001618671417236328,
-      "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0001,
-      "reward": 0.04508928821887821,
-      "reward_std": 0.07083694078028202,
-      "rewards/accuracy_reward": 0.04508928821887821,
+      "completion_length": 736.6721374511719,
+      "epoch": 0.128,
+      "grad_norm": 0.36135947704315186,
+      "kl": 0.0037824630737304686,
+      "learning_rate": 2.993961440992859e-06,
+      "loss": 0.0002,
+      "reward": 0.1305803638882935,
+      "reward_std": 0.12962731290608645,
+      "rewards/accuracy_reward": 0.1305803638882935,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
-      "completion_length": 755.3701187133789,
-      "epoch": 0.255863539445629,
-      "grad_norm": 0.7283040285110474,
-      "kl": 0.00929851531982422,
-      "learning_rate": 2.7836719084521715e-06,
+      "completion_length": 779.8250381469727,
+      "epoch": 0.192,
+      "grad_norm": 1.732832670211792,
+      "kl": 0.010242462158203125,
+      "learning_rate": 2.9265847744427307e-06,
       "loss": 0.0004,
-      "reward": 0.061830359417945145,
-      "reward_std": 0.08021324193105102,
-      "rewards/accuracy_reward": 0.061830359417945145,
+      "reward": 0.1544642923399806,
+      "reward_std": 0.1452927845530212,
+      "rewards/accuracy_reward": 0.1544642923399806,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
-      "completion_length": 759.1250411987305,
-      "epoch": 0.3411513859275053,
-      "grad_norm": 0.9553681015968323,
-      "kl": 0.011894989013671874,
-      "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0005,
-      "reward": 0.07767857508733869,
-      "reward_std": 0.09344197846949101,
-      "rewards/accuracy_reward": 0.07767857508733869,
+      "completion_length": 793.7176681518555,
+      "epoch": 0.256,
+      "grad_norm": 0.45336416363716125,
+      "kl": 0.05513496398925781,
+      "learning_rate": 2.7876731904027993e-06,
+      "loss": 0.0022,
+      "reward": 0.1392857213038951,
+      "reward_std": 0.1322711819317192,
+      "rewards/accuracy_reward": 0.1392857213038951,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 786.6553939819336,
-      "epoch": 0.42643923240938164,
-      "grad_norm": 0.3153317868709564,
-      "kl": 0.0048351287841796875,
-      "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0002,
-      "reward": 0.23928572454024105,
-      "reward_std": 0.2163457485847175,
-      "rewards/accuracy_reward": 0.23928572454024105,
+      "completion_length": 772.2924423217773,
+      "epoch": 0.32,
+      "grad_norm": 1.4548314809799194,
+      "kl": 0.008623123168945312,
+      "learning_rate": 2.584192295741087e-06,
+      "loss": 0.0003,
+      "reward": 0.17254465222358703,
+      "reward_std": 0.1551555698737502,
+      "rewards/accuracy_reward": 0.17254465222358703,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 800.1018218994141,
-      "epoch": 0.511727078891258,
-      "grad_norm": 0.23729673027992249,
-      "kl": 0.005898094177246094,
-      "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0002,
-      "reward": 0.2857142981141806,
-      "reward_std": 0.2412072943523526,
-      "rewards/accuracy_reward": 0.2857142981141806,
+      "completion_length": 758.8147674560547,
+      "epoch": 0.384,
+      "grad_norm": 0.43251317739486694,
+      "kl": 0.04488029479980469,
+      "learning_rate": 2.3263454721781537e-06,
+      "loss": 0.0018,
+      "reward": 0.18013393748551607,
+      "reward_std": 0.15790632385760545,
+      "rewards/accuracy_reward": 0.18013393748551607,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 809.353158569336,
-      "epoch": 0.5970149253731343,
-      "grad_norm": 0.29627376794815063,
-      "kl": 0.003314781188964844,
-      "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0001,
-      "reward": 0.3125000149011612,
-      "reward_std": 0.24806460849940776,
-      "rewards/accuracy_reward": 0.3125000149011612,
+      "completion_length": 759.0868667602539,
+      "epoch": 0.448,
+      "grad_norm": 2.017112970352173,
+      "kl": 0.01151580810546875,
+      "learning_rate": 2.027062236122014e-06,
+      "loss": 0.0005,
+      "reward": 0.16049108020961284,
+      "reward_std": 0.14527264153584837,
+      "rewards/accuracy_reward": 0.16049108020961284,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 788.3007049560547,
-      "epoch": 0.6823027718550106,
-      "grad_norm": 0.11735204607248306,
-      "kl": 0.003574562072753906,
-      "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0001,
-      "reward": 0.2823660859838128,
-      "reward_std": 0.20102398535236715,
-      "rewards/accuracy_reward": 0.2823660859838128,
+      "completion_length": 762.179719543457,
+      "epoch": 0.512,
+      "grad_norm": 2.426039934158325,
+      "kl": 0.0172943115234375,
+      "learning_rate": 1.7013498987264833e-06,
+      "loss": 0.0007,
+      "reward": 0.17991072311997414,
+      "reward_std": 0.15640948014333844,
+      "rewards/accuracy_reward": 0.17991072311997414,
       "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 780.8183395385743,
-      "epoch": 0.767590618336887,
-      "grad_norm": 0.9417461156845093,
-      "kl": 0.0042591094970703125,
-      "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0002,
-      "reward": 0.2698660843074322,
-      "reward_std": 0.1852410165593028,
-      "rewards/accuracy_reward": 0.2698660843074322,
+      "completion_length": 754.7252563476562,
+      "epoch": 0.576,
+      "grad_norm": 1.071414828300476,
+      "kl": 0.008959579467773437,
+      "learning_rate": 1.3655410366448499e-06,
+      "loss": 0.0004,
+      "reward": 0.20066965445876123,
+      "reward_std": 0.15480793919414282,
+      "rewards/accuracy_reward": 0.20066965445876123,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
-      "completion_length": 874.1305999755859,
-      "epoch": 0.8528784648187633,
-      "grad_norm": NaN,
-      "kl": NaN,
-      "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0103,
-      "reward": 0.08928571869619191,
-      "reward_std": 0.0756364663131535,
-      "rewards/accuracy_reward": 0.08928571869619191,
+      "completion_length": 765.8024856567383,
+      "epoch": 0.64,
+      "grad_norm": 0.8965884447097778,
+      "kl": 0.008967971801757813,
+      "learning_rate": 1.036474508437579e-06,
+      "loss": 0.0004,
+      "reward": 0.18415179559960962,
+      "reward_std": 0.17140543824061752,
+      "rewards/accuracy_reward": 0.18415179559960962,
       "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 1024.0,
-      "epoch": 0.9381663113006397,
-      "grad_norm": NaN,
-      "kl": NaN,
-      "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
-      "rewards/accuracy_reward": 0.0,
+      "completion_length": 740.766552734375,
+      "epoch": 0.704,
+      "grad_norm": 27.7662296295166,
+      "kl": 0.016036224365234376,
+      "learning_rate": 7.30651083891141e-07,
+      "loss": 0.0006,
+      "reward": 0.1348214357160032,
+      "reward_std": 0.1273266337811947,
+      "rewards/accuracy_reward": 0.1348214357160032,
       "rewards/format_reward": 0.0,
       "step": 55
     },
     {
-      "completion_length": 1024.0,
-      "epoch": 0.9893390191897654,
-      "kl": NaN,
-      "reward": 0.0,
-      "reward_std": 0.0,
-      "rewards/accuracy_reward": 0.0,
+      "completion_length": 741.5098526000977,
+      "epoch": 0.768,
+      "grad_norm": 0.12588420510292053,
+      "kl": 0.011777877807617188,
+      "learning_rate": 4.63406026519703e-07,
+      "loss": 0.0005,
+      "reward": 0.19330358118750154,
+      "reward_std": 0.15862846123054625,
+      "rewards/accuracy_reward": 0.19330358118750154,
+      "rewards/format_reward": 0.0,
+      "step": 60
+    },
+    {
+      "completion_length": 734.3199005126953,
+      "epoch": 0.832,
+      "grad_norm": 4.302868843078613,
+      "kl": 0.07119407653808593,
+      "learning_rate": 2.48140119418046e-07,
+      "loss": 0.0029,
+      "reward": 0.16562500977888703,
+      "reward_std": 0.14798311032354833,
+      "rewards/accuracy_reward": 0.16562500977888703,
+      "rewards/format_reward": 0.0,
+      "step": 65
+    },
+    {
+      "completion_length": 737.8087387084961,
+      "epoch": 0.896,
+      "grad_norm": 1.6815804243087769,
+      "kl": 0.00724334716796875,
+      "learning_rate": 9.564769404039419e-08,
+      "loss": 0.0003,
+      "reward": 0.17589286509901286,
+      "reward_std": 0.14854512372985482,
+      "rewards/accuracy_reward": 0.17589286509901286,
+      "rewards/format_reward": 0.0,
+      "step": 70
+    },
+    {
+      "completion_length": 740.4969161987304,
+      "epoch": 0.96,
+      "grad_norm": 0.14792795479297638,
+      "kl": 0.034131622314453124,
+      "learning_rate": 1.357535734809795e-08,
+      "loss": 0.0014,
+      "reward": 0.16607143636792898,
+      "reward_std": 0.14445759197697045,
+      "rewards/accuracy_reward": 0.16607143636792898,
+      "rewards/format_reward": 0.0,
+      "step": 75
+    },
+    {
+      "completion_length": 740.5067342122396,
+      "epoch": 0.9984,
+      "kl": 0.02994537353515625,
+      "reward": 0.17708334047347307,
+      "reward_std": 0.1565772045093278,
+      "rewards/accuracy_reward": 0.17708334047347307,
       "rewards/format_reward": 0.0,
-      "step": 58,
+      "step": 78,
       "total_flos": 0.0,
-      "train_loss": 0.0010395098450702472,
-      "train_runtime": 14559.9882,
-      "train_samples_per_second": 0.515,
+      "train_loss": 0.0006319970209619013,
+      "train_runtime": 19204.1352,
+      "train_samples_per_second": 0.521,
       "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 5,
-  "max_steps": 58,
+  "max_steps": 78,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e015d8ab2e5cb45b3da98676b2843655934f11c70a34a51a48328776a67f59c5
+oid sha256:87d6b88ba5aa2bf4c81fdcad1bb5a213d7285d62b4a660d9d975034da8f3cab7
 size 7992