Model save

README.md

@@ -26,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/build_r1/huggingface/runs/521r385l) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/build_r1/huggingface/runs/sbd67t58) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
@@ -34,7 +34,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.16.0.dev0
-- Transformers: 4.50.0.dev0
+- Transformers: 4.49.0
 - Pytorch: 2.5.1+cu124
 - Datasets: 3.3.1
 - Tokenizers: 0.21.0

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.041315901221643234,
-    "train_runtime": 6824.0967,
+    "train_loss": 0.03965516161473318,
+    "train_runtime": 10204.823,
     "train_samples": 7500,
-    "train_samples_per_second": 1.099,
-    "train_steps_per_second": 0.008
+    "train_samples_per_second": 1.102,
+    "train_steps_per_second": 0.009
 }

config.json

@@ -22,7 +22,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.50.0.dev0",
+  "transformers_version": "4.49.0",
   "use_cache": false,
   "use_mrope": false,
   "use_sliding_window": false,

generation_config.json

@@ -2,5 +2,5 @@
   "bos_token_id": 151643,
   "eos_token_id": 151643,
   "max_new_tokens": 2048,
-  "transformers_version": "4.50.0.dev0"
+  "transformers_version": "4.49.0"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1276b52a444f36ce7a2438ec8eb171d7a6c10936b36fe98759917253d63a8344
+oid sha256:6f5229b390a6d5a3c821d1b16005e5c679c73020aad6a61f898cc4c752e570b7
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0325bb74df5b1b59a00cb2724a66404e2c614ac3f34121754f1585e92d219a4
+oid sha256:1b74dbe92920694029d0c604f0890522acb2ece89f0518a5a9f420124b837b40
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ce7c8740a88cbc6ddc98aab33ea1a20f5a7107c2934056f8c8b8c22faebaa0d
+oid sha256:5d323dcf810106f79f0588f9bc36a276f5414fbe7bb01853a65a2fee4eb10885
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:322362afbb95734a973ce1fa95a7b55ced779dedfe1b251ea70175bf54c1f2d8
+oid sha256:9d3c27c38871f8ffbdcc74752d141b37109625c30769f8104121ce07941c513a
 size 1089994880

tokenizer_config.json

@@ -202,7 +202,6 @@
   "extra_special_tokens": {},
   "model_max_length": 131072,
   "pad_token": "<|endoftext|>",
-  "padding_side": "left",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.041315901221643234,
-    "train_runtime": 6824.0967,
+    "train_loss": 0.03965516161473318,
+    "train_runtime": 10204.823,
     "train_samples": 7500,
-    "train_samples_per_second": 1.099,
-    "train_steps_per_second": 0.008
+    "train_samples_per_second": 1.102,
+    "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -1,189 +1,267 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9893390191897654,
+  "epoch": 1.4946695095948828,
   "eval_steps": 100,
-  "global_step": 58,
+  "global_step": 87,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 602.1573867797852,
+      "completion_length": 605.1830520629883,
       "epoch": 0.017057569296375266,
-      "grad_norm": 0.49788331985473633,
+      "grad_norm": 0.5403730869293213,
       "kl": 0.0,
-      "learning_rate": 5e-07,
-      "loss": 0.0235,
-      "reward": 0.6395089626312256,
-      "reward_std": 0.34354935958981514,
-      "rewards/accuracy_reward": 0.6339286044239998,
+      "learning_rate": 3.333333333333333e-07,
+      "loss": 0.0308,
+      "reward": 0.631696455180645,
+      "reward_std": 0.3498076871037483,
+      "rewards/accuracy_reward": 0.6261160969734192,
       "rewards/format_reward": 0.005580357392318547,
       "step": 1
     },
     {
-      "completion_length": 602.9565010070801,
+      "completion_length": 603.803879737854,
       "epoch": 0.08528784648187633,
-      "grad_norm": 0.31691527366638184,
-      "kl": 0.00025323033332824707,
-      "learning_rate": 2.5e-06,
-      "loss": 0.0201,
-      "reward": 0.6130022583529353,
-      "reward_std": 0.36007128469645977,
-      "rewards/accuracy_reward": 0.6104910988360643,
-      "rewards/format_reward": 0.002511160826543346,
+      "grad_norm": 1.4775316715240479,
+      "kl": 0.00015020370483398438,
+      "learning_rate": 1.6666666666666669e-06,
+      "loss": 0.0283,
+      "reward": 0.6054687770083547,
+      "reward_std": 0.35884954407811165,
+      "rewards/accuracy_reward": 0.6046317219734192,
+      "rewards/format_reward": 0.000837053608847782,
       "step": 5
     },
     {
-      "completion_length": 604.4129737854004,
+      "completion_length": 606.0585052490235,
       "epoch": 0.17057569296375266,
-      "grad_norm": 1.7574856281280518,
-      "kl": 0.004290962219238281,
-      "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0526,
-      "reward": 0.6930803835391999,
-      "reward_std": 0.30544489361345767,
-      "rewards/accuracy_reward": 0.6915178835391999,
-      "rewards/format_reward": 0.0015625000698491931,
+      "grad_norm": 0.26707330346107483,
+      "kl": 0.001517653465270996,
+      "learning_rate": 2.9987834972573546e-06,
+      "loss": 0.0458,
+      "reward": 0.6533482439815999,
+      "reward_std": 0.32249790802598,
+      "rewards/accuracy_reward": 0.652455386519432,
+      "rewards/format_reward": 0.0008928571827709675,
       "step": 10
     },
     {
-      "completion_length": 600.6560562133789,
+      "completion_length": 607.5294906616211,
       "epoch": 0.255863539445629,
-      "grad_norm": 0.1719939261674881,
-      "kl": 0.015668106079101563,
-      "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0603,
-      "reward": 0.7658482477068901,
-      "reward_std": 0.2067408435046673,
-      "rewards/accuracy_reward": 0.7656250327825547,
+      "grad_norm": 0.13552981615066528,
+      "kl": 0.006855583190917969,
+      "learning_rate": 2.956412726139078e-06,
+      "loss": 0.0701,
+      "reward": 0.7529018178582192,
+      "reward_std": 0.22657863702625036,
+      "rewards/accuracy_reward": 0.7526786029338837,
       "rewards/format_reward": 0.00022321429569274187,
       "step": 15
     },
     {
-      "completion_length": 594.912075805664,
+      "completion_length": 592.2439987182618,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.5162317752838135,
-      "kl": 0.005500221252441406,
-      "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0549,
-      "reward": 0.7725446730852127,
-      "reward_std": 0.2103592725470662,
-      "rewards/accuracy_reward": 0.7723214596509933,
-      "rewards/format_reward": 0.00022321429569274187,
+      "grad_norm": 2.5956568717956543,
+      "kl": 0.014976119995117188,
+      "learning_rate": 2.8551756519155732e-06,
+      "loss": 0.0501,
+      "reward": 0.7772321775555611,
+      "reward_std": 0.18385109901428223,
+      "rewards/accuracy_reward": 0.7772321775555611,
+      "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 602.0649864196778,
+      "completion_length": 595.9457809448243,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.1503874659538269,
-      "kl": 0.004841995239257812,
-      "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0425,
-      "reward": 0.7611607491970063,
-      "reward_std": 0.19463529847562314,
-      "rewards/accuracy_reward": 0.7609375342726707,
-      "rewards/format_reward": 0.00022321429569274187,
+      "grad_norm": 0.31570613384246826,
+      "kl": 0.004840660095214844,
+      "learning_rate": 2.699164145105252e-06,
+      "loss": 0.0432,
+      "reward": 0.7618303894996643,
+      "reward_std": 0.1812900934368372,
+      "rewards/accuracy_reward": 0.7618303894996643,
+      "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 600.5692260742187,
+      "completion_length": 598.4087356567383,
       "epoch": 0.511727078891258,
-      "grad_norm": 0.111796073615551,
-      "kl": 0.006829071044921875,
-      "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0364,
-      "reward": 0.7482143223285675,
-      "reward_std": 0.18839258402585984,
-      "rewards/accuracy_reward": 0.7482143223285675,
+      "grad_norm": 0.09540420770645142,
+      "kl": 0.0038333892822265624,
+      "learning_rate": 2.4946839873611927e-06,
+      "loss": 0.0415,
+      "reward": 0.7622768223285675,
+      "reward_std": 0.18161814119666814,
+      "rewards/accuracy_reward": 0.7622768223285675,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 595.0308326721191,
+      "completion_length": 587.7656509399415,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.6281359791755676,
-      "kl": 0.27039794921875,
-      "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0478,
-      "reward": 0.7406250335276127,
-      "reward_std": 0.18172951051965355,
-      "rewards/accuracy_reward": 0.7401786051690579,
-      "rewards/format_reward": 0.00044642859138548373,
+      "grad_norm": 0.10136830061674118,
+      "kl": 0.005519866943359375,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0319,
+      "reward": 0.7529018253087998,
+      "reward_std": 0.1641070661135018,
+      "rewards/accuracy_reward": 0.7529018253087998,
+      "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 581.8551567077636,
+      "completion_length": 576.0547142028809,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.29348960518836975,
-      "kl": 0.045685958862304685,
-      "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0408,
-      "reward": 0.7723214700818062,
-      "reward_std": 0.1774477436207235,
-      "rewards/accuracy_reward": 0.7716518267989159,
-      "rewards/format_reward": 0.0006696428870782256,
+      "grad_norm": 0.2535816729068756,
+      "kl": 0.005191802978515625,
+      "learning_rate": 1.975001990702209e-06,
+      "loss": 0.0346,
+      "reward": 0.7754464656114578,
+      "reward_std": 0.1725513377226889,
+      "rewards/accuracy_reward": 0.7754464656114578,
+      "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 597.0538185119628,
+      "completion_length": 592.9451164245605,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.13345518708229065,
-      "kl": 0.01571235656738281,
-      "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0281,
-      "reward": 0.7609375342726707,
-      "reward_std": 0.17938947193324567,
-      "rewards/accuracy_reward": 0.7598214626312256,
-      "rewards/format_reward": 0.0011160714784637094,
+      "grad_norm": 0.12311802059412003,
+      "kl": 0.004965591430664063,
+      "learning_rate": 1.6808050203829845e-06,
+      "loss": 0.0432,
+      "reward": 0.7558036044239997,
+      "reward_std": 0.19019564976915718,
+      "rewards/accuracy_reward": 0.7555803909897805,
+      "rewards/format_reward": 0.00022321429569274187,
       "step": 45
     },
     {
-      "completion_length": 604.6993606567382,
+      "completion_length": 588.0194488525391,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.24237532913684845,
-      "kl": 0.005515289306640625,
-      "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0334,
-      "reward": 0.7473214641213417,
-      "reward_std": 0.19217551834881305,
-      "rewards/accuracy_reward": 0.7455357491970063,
-      "rewards/format_reward": 0.001785714365541935,
+      "grad_norm": 0.10202702134847641,
+      "kl": 0.005793380737304688,
+      "learning_rate": 1.3793001469249112e-06,
+      "loss": 0.0278,
+      "reward": 0.7582589641213417,
+      "reward_std": 0.17491137199103832,
+      "rewards/accuracy_reward": 0.7582589641213417,
+      "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 585.3790458679199,
+      "completion_length": 577.1547119140625,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.21673214435577393,
-      "kl": 0.0057525634765625,
-      "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0363,
-      "reward": 0.7837053894996643,
-      "reward_std": 0.18726392211392523,
-      "rewards/accuracy_reward": 0.7819196790456772,
-      "rewards/format_reward": 0.001785714365541935,
+      "grad_norm": 0.09594480693340302,
+      "kl": 0.00527496337890625,
+      "learning_rate": 1.0826738041253211e-06,
+      "loss": 0.0349,
+      "reward": 0.7921875387430191,
+      "reward_std": 0.1892126789316535,
+      "rewards/accuracy_reward": 0.7917411103844643,
+      "rewards/format_reward": 0.00044642859138548373,
       "step": 55
     },
     {
-      "completion_length": 580.9449704488119,
-      "epoch": 0.9893390191897654,
-      "kl": 0.0047308603922526045,
-      "reward": 0.7645089675982794,
-      "reward_std": 0.18809200543910265,
-      "rewards/accuracy_reward": 0.763392892976602,
-      "rewards/format_reward": 0.0011160714784637094,
-      "step": 58,
+      "completion_length": 568.16053425182,
+      "epoch": 1.0341151385927505,
+      "grad_norm": 0.08510848134756088,
+      "kl": 0.005277807062322443,
+      "learning_rate": 8.029152419343472e-07,
+      "loss": 0.0384,
+      "reward": 0.7849026335911318,
+      "reward_std": 0.16940699830989947,
+      "rewards/accuracy_reward": 0.7849026335911318,
+      "rewards/format_reward": 0.0,
+      "step": 60
+    },
+    {
+      "completion_length": 560.2404289245605,
+      "epoch": 1.1194029850746268,
+      "grad_norm": 0.10467664897441864,
+      "kl": 0.006550979614257812,
+      "learning_rate": 5.513319366069343e-07,
+      "loss": 0.0334,
+      "reward": 0.7776786103844643,
+      "reward_std": 0.17760842395946383,
+      "rewards/accuracy_reward": 0.7776786103844643,
+      "rewards/format_reward": 0.0,
+      "step": 65
+    },
+    {
+      "completion_length": 555.7098434448242,
+      "epoch": 1.2046908315565032,
+      "grad_norm": 0.07688478380441666,
+      "kl": 0.005760955810546875,
+      "learning_rate": 3.380925572585183e-07,
+      "loss": 0.0404,
+      "reward": 0.784598246216774,
+      "reward_std": 0.16191664077341555,
+      "rewards/accuracy_reward": 0.784598246216774,
+      "rewards/format_reward": 0.0,
+      "step": 70
+    },
+    {
+      "completion_length": 572.1274841308593,
+      "epoch": 1.2899786780383795,
+      "grad_norm": 0.09251231700181961,
+      "kl": 0.0058074951171875,
+      "learning_rate": 1.718159615201853e-07,
+      "loss": 0.0414,
+      "reward": 0.7857143238186837,
+      "reward_std": 0.1789614163339138,
+      "rewards/accuracy_reward": 0.7854911088943481,
+      "rewards/format_reward": 0.00022321429569274187,
+      "step": 75
+    },
+    {
+      "completion_length": 555.3935531616211,
+      "epoch": 1.375266524520256,
+      "grad_norm": 0.09592189639806747,
+      "kl": 0.005047607421875,
+      "learning_rate": 5.922283255294164e-08,
+      "loss": 0.0362,
+      "reward": 0.7993303924798966,
+      "reward_std": 0.15318573899567128,
+      "rewards/accuracy_reward": 0.7993303924798966,
+      "rewards/format_reward": 0.0,
+      "step": 80
+    },
+    {
+      "completion_length": 563.471452331543,
+      "epoch": 1.4605543710021323,
+      "grad_norm": 0.08133210241794586,
+      "kl": 0.005420303344726563,
+      "learning_rate": 4.864037798685106e-09,
+      "loss": 0.0287,
+      "reward": 0.7917411044239998,
+      "reward_std": 0.16233704406768085,
+      "rewards/accuracy_reward": 0.7917411044239998,
+      "rewards/format_reward": 0.0,
+      "step": 85
+    },
+    {
+      "completion_length": 601.3694458007812,
+      "epoch": 1.4946695095948828,
+      "kl": 0.004954338073730469,
+      "reward": 0.7209821734577417,
+      "reward_std": 0.17686136066913605,
+      "rewards/accuracy_reward": 0.7209821734577417,
+      "rewards/format_reward": 0.0,
+      "step": 87,
       "total_flos": 0.0,
-      "train_loss": 0.041315901221643234,
-      "train_runtime": 6824.0967,
-      "train_samples_per_second": 1.099,
-      "train_steps_per_second": 0.008
+      "train_loss": 0.03965516161473318,
+      "train_runtime": 10204.823,
+      "train_samples_per_second": 1.102,
+      "train_steps_per_second": 0.009
     }
   ],
   "logging_steps": 5,
-  "max_steps": 58,
+  "max_steps": 87,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20161f39d1b21a3a3a053fb61dc4ba40dda709060888f19e8d7bf40e2ee58015
+oid sha256:0ba803b2ace34fcfc3c61960d2aa18dc3b43a1ad62eedcaa1dfdd55f3db3a693
 size 7928