Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: openai-community/gpt2
-datasets: HuggingFaceFW/fineweb-edu
 library_name: transformers
 model_name: gpt2-RMT-2
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - sft
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for gpt2-RMT-2
 
-This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceFW/fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/rtv4pjln) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/xxpt9481) 
 
 
 This model was trained with SFT.

all_results.json

@@ -4,9 +4,9 @@
     "eval_samples_per_second": 198.662,
     "eval_steps_per_second": 12.416,
     "total_flos": 5419400756723712.0,
-    "train_loss": 3.61015625,
-    "train_runtime": 338.5176,
+    "train_loss": 3.609872159090909,
+    "train_runtime": 919.7134,
     "train_samples": 19883,
-    "train_samples_per_second": 31.227,
-    "train_steps_per_second": 0.975
+    "train_samples_per_second": 11.494,
+    "train_steps_per_second": 0.359
 }

config.json

@@ -33,6 +33,6 @@
   },
   "torch_dtype": "bfloat16",
   "transformers_version": "4.50.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 50257
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1422a31ad724b5784e66d973fec1851ddb469e716474476e44603c91f8e30ca7
+oid sha256:585da17beba27ab616789ca4c8a12e9173b873bedfa95a97b6ec57e2b190642b
 size 248912768

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 5419400756723712.0,
-    "train_loss": 3.61015625,
-    "train_runtime": 338.5176,
+    "train_loss": 3.609872159090909,
+    "train_runtime": 919.7134,
     "train_samples": 19883,
-    "train_samples_per_second": 31.227,
-    "train_steps_per_second": 0.975
+    "train_samples_per_second": 11.494,
+    "train_steps_per_second": 0.359
 }

trainer_state.json

@@ -10,297 +10,297 @@
   "log_history": [
     {
       "epoch": 0.030257186081694403,
-      "grad_norm": 53.20387628855397,
+      "grad_norm": 52.88699541505755,
       "learning_rate": 6.060606060606061e-06,
-      "loss": 7.3984,
-      "mean_token_accuracy": 0.1354332946240902,
+      "loss": 7.4,
+      "mean_token_accuracy": 0.13546177372336388,
       "step": 10
     },
     {
       "epoch": 0.060514372163388806,
-      "grad_norm": 27.17948871011392,
+      "grad_norm": 26.843614632955404,
       "learning_rate": 1.2121212121212122e-05,
-      "loss": 6.7727,
-      "mean_token_accuracy": 0.14481190741062164,
+      "loss": 6.8156,
+      "mean_token_accuracy": 0.14476052597165107,
       "step": 20
     },
     {
       "epoch": 0.0907715582450832,
-      "grad_norm": 9.47619461823253,
+      "grad_norm": 9.852827567927251,
       "learning_rate": 1.8181818181818182e-05,
-      "loss": 5.5984,
-      "mean_token_accuracy": 0.17588953599333762,
+      "loss": 5.6055,
+      "mean_token_accuracy": 0.17639728486537934,
       "step": 30
     },
     {
       "epoch": 0.12102874432677761,
-      "grad_norm": 4.801175322490429,
+      "grad_norm": 4.231145666737617,
       "learning_rate": 1.9972599751485225e-05,
-      "loss": 4.6516,
-      "mean_token_accuracy": 0.24421471059322358,
+      "loss": 4.6,
+      "mean_token_accuracy": 0.2488641396164894,
       "step": 40
     },
     {
       "epoch": 0.15128593040847202,
-      "grad_norm": 3.381736221106582,
+      "grad_norm": 3.2271685140325594,
       "learning_rate": 1.9838755799290993e-05,
-      "loss": 4.009,
-      "mean_token_accuracy": 0.3037181183695793,
+      "loss": 3.9941,
+      "mean_token_accuracy": 0.3051066160202026,
       "step": 50
     },
     {
       "epoch": 0.1815431164901664,
-      "grad_norm": 1.8196128883676905,
+      "grad_norm": 1.885693756577763,
       "learning_rate": 1.9594929736144978e-05,
-      "loss": 3.5969,
-      "mean_token_accuracy": 0.3466367706656456,
+      "loss": 3.609,
+      "mean_token_accuracy": 0.34526345133781433,
       "step": 60
     },
     {
       "epoch": 0.2118003025718608,
-      "grad_norm": 1.0989833139359242,
+      "grad_norm": 1.1244069323463834,
       "learning_rate": 1.9243847161266924e-05,
-      "loss": 3.3781,
-      "mean_token_accuracy": 0.37338929772377016,
+      "loss": 3.3801,
+      "mean_token_accuracy": 0.37311544418334963,
       "step": 70
     },
     {
       "epoch": 0.24205748865355523,
-      "grad_norm": 0.9673089083213241,
+      "grad_norm": 0.9867146004226443,
       "learning_rate": 1.8789432636206197e-05,
-      "loss": 3.325,
-      "mean_token_accuracy": 0.37947496473789216,
+      "loss": 3.3258,
+      "mean_token_accuracy": 0.3792398452758789,
       "step": 80
     },
     {
       "epoch": 0.2723146747352496,
-      "grad_norm": 0.8526479435566983,
+      "grad_norm": 0.8107915407779778,
       "learning_rate": 1.8236765814298328e-05,
-      "loss": 3.2707,
-      "mean_token_accuracy": 0.3870904952287674,
+      "loss": 3.2719,
+      "mean_token_accuracy": 0.38700797259807584,
       "step": 90
     },
     {
       "epoch": 0.30257186081694404,
-      "grad_norm": 0.8771420630730227,
+      "grad_norm": 0.8941659318929186,
       "learning_rate": 1.7592024657977432e-05,
-      "loss": 3.2793,
-      "mean_token_accuracy": 0.3854751497507095,
+      "loss": 3.2785,
+      "mean_token_accuracy": 0.38543802350759504,
       "step": 100
     },
     {
       "epoch": 0.30257186081694404,
-      "eval_runtime": 0.2258,
-      "eval_samples_per_second": 212.573,
-      "eval_steps_per_second": 13.286,
+      "eval_runtime": 0.237,
+      "eval_samples_per_second": 202.542,
+      "eval_steps_per_second": 12.659,
       "step": 100
     },
     {
       "epoch": 0.3328290468986384,
-      "grad_norm": 0.7981055028496323,
+      "grad_norm": 0.7913271033960183,
       "learning_rate": 1.686241637868734e-05,
-      "loss": 3.2504,
-      "mean_token_accuracy": 0.3904677391052246,
+      "loss": 3.25,
+      "mean_token_accuracy": 0.3904023408889771,
       "step": 110
     },
     {
       "epoch": 0.3630862329803328,
-      "grad_norm": 0.7170465908499676,
+      "grad_norm": 0.7385462564687278,
       "learning_rate": 1.6056096871376667e-05,
-      "loss": 3.277,
-      "mean_token_accuracy": 0.3867432788014412,
+      "loss": 3.2746,
+      "mean_token_accuracy": 0.3869674876332283,
       "step": 120
     },
     {
       "epoch": 0.39334341906202724,
-      "grad_norm": 0.7497925109759247,
+      "grad_norm": 0.7171108615447848,
       "learning_rate": 1.5182079544175957e-05,
       "loss": 3.2488,
-      "mean_token_accuracy": 0.39058607518672944,
+      "mean_token_accuracy": 0.39062655717134476,
       "step": 130
     },
     {
       "epoch": 0.4236006051437216,
-      "grad_norm": 0.7800876699148922,
+      "grad_norm": 0.7917794437598532,
       "learning_rate": 1.4250134562400301e-05,
-      "loss": 3.2242,
-      "mean_token_accuracy": 0.392949678003788,
+      "loss": 3.2227,
+      "mean_token_accuracy": 0.39324862957000734,
       "step": 140
     },
     {
       "epoch": 0.45385779122541603,
-      "grad_norm": 0.7636894413901015,
+      "grad_norm": 0.7618905726241425,
       "learning_rate": 1.3270679633174219e-05,
-      "loss": 3.2141,
-      "mean_token_accuracy": 0.3948305919766426,
+      "loss": 3.2133,
+      "mean_token_accuracy": 0.39474184066057205,
       "step": 150
     },
     {
       "epoch": 0.48411497730711045,
-      "grad_norm": 0.7028258888259162,
+      "grad_norm": 0.710429734736302,
       "learning_rate": 1.2254663551538047e-05,
-      "loss": 3.1902,
-      "mean_token_accuracy": 0.39704713225364685,
+      "loss": 3.1895,
+      "mean_token_accuracy": 0.3969797551631927,
       "step": 160
     },
     {
       "epoch": 0.5143721633888049,
-      "grad_norm": 0.7254535951054281,
+      "grad_norm": 0.7314245404422203,
       "learning_rate": 1.121344380981082e-05,
       "loss": 3.2254,
-      "mean_token_accuracy": 0.3936114251613617,
+      "mean_token_accuracy": 0.39376401603221894,
       "step": 170
     },
     {
       "epoch": 0.5446293494704992,
-      "grad_norm": 0.7313179656460369,
+      "grad_norm": 0.711095968768027,
       "learning_rate": 1.015865963834808e-05,
-      "loss": 3.1965,
-      "mean_token_accuracy": 0.39705405980348585,
+      "loss": 3.1969,
+      "mean_token_accuracy": 0.39729384928941724,
       "step": 180
     },
     {
       "epoch": 0.5748865355521936,
-      "grad_norm": 0.6925411391649295,
+      "grad_norm": 0.7048692336249331,
       "learning_rate": 9.102101896903084e-06,
       "loss": 3.2078,
-      "mean_token_accuracy": 0.3967175245285034,
+      "mean_token_accuracy": 0.3967027798295021,
       "step": 190
     },
     {
       "epoch": 0.6051437216338881,
-      "grad_norm": 0.7015683467230908,
+      "grad_norm": 0.714339105324786,
       "learning_rate": 8.055581271005292e-06,
-      "loss": 3.209,
-      "mean_token_accuracy": 0.39455645680427553,
+      "loss": 3.2086,
+      "mean_token_accuracy": 0.3946896642446518,
       "step": 200
     },
     {
       "epoch": 0.6051437216338881,
-      "eval_runtime": 0.2238,
-      "eval_samples_per_second": 214.446,
-      "eval_steps_per_second": 13.403,
+      "eval_runtime": 0.2292,
+      "eval_samples_per_second": 209.454,
+      "eval_steps_per_second": 13.091,
       "step": 200
     },
     {
       "epoch": 0.6354009077155824,
-      "grad_norm": 0.726599375455085,
+      "grad_norm": 0.732439534421158,
       "learning_rate": 7.0307962467172555e-06,
-      "loss": 3.1992,
-      "mean_token_accuracy": 0.3978341415524483,
+      "loss": 3.198,
+      "mean_token_accuracy": 0.39805368781089784,
       "step": 210
     },
     {
       "epoch": 0.6656580937972768,
-      "grad_norm": 0.6917092811454406,
+      "grad_norm": 0.7679520615376684,
       "learning_rate": 6.039202339608432e-06,
-      "loss": 3.202,
-      "mean_token_accuracy": 0.3954534113407135,
+      "loss": 3.2008,
+      "mean_token_accuracy": 0.39553126841783526,
       "step": 220
     },
     {
       "epoch": 0.6959152798789713,
-      "grad_norm": 0.6705832721497097,
+      "grad_norm": 0.6598246730886332,
       "learning_rate": 5.091884039764321e-06,
-      "loss": 3.1969,
-      "mean_token_accuracy": 0.3959610104560852,
+      "loss": 3.1961,
+      "mean_token_accuracy": 0.3962506249547005,
       "step": 230
     },
     {
       "epoch": 0.7261724659606656,
-      "grad_norm": 0.7043055334462106,
+      "grad_norm": 0.6582768941498197,
       "learning_rate": 4.19943090428802e-06,
-      "loss": 3.2098,
-      "mean_token_accuracy": 0.39361765086650846,
+      "loss": 3.2105,
+      "mean_token_accuracy": 0.3936332181096077,
       "step": 240
     },
     {
       "epoch": 0.75642965204236,
-      "grad_norm": 0.7337550100582009,
+      "grad_norm": 0.7382830086371551,
       "learning_rate": 3.37181918238904e-06,
       "loss": 3.207,
-      "mean_token_accuracy": 0.3955203667283058,
+      "mean_token_accuracy": 0.39551725089550016,
       "step": 250
     },
     {
       "epoch": 0.7866868381240545,
-      "grad_norm": 0.6893800072740404,
+      "grad_norm": 0.6780081878254819,
       "learning_rate": 2.618300296308135e-06,
-      "loss": 3.2207,
-      "mean_token_accuracy": 0.3941065609455109,
+      "loss": 3.2203,
+      "mean_token_accuracy": 0.393978887796402,
       "step": 260
     },
     {
       "epoch": 0.8169440242057489,
-      "grad_norm": 0.674883610377537,
+      "grad_norm": 0.6887657468672304,
       "learning_rate": 1.947297424689414e-06,
-      "loss": 3.1988,
-      "mean_token_accuracy": 0.3957181140780449,
+      "loss": 3.1984,
+      "mean_token_accuracy": 0.39577416330575943,
       "step": 270
     },
     {
       "epoch": 0.8472012102874432,
-      "grad_norm": 0.6757567363771522,
+      "grad_norm": 0.6760254781738906,
       "learning_rate": 1.3663113444380905e-06,
       "loss": 3.2012,
-      "mean_token_accuracy": 0.39344170689582825,
+      "mean_token_accuracy": 0.39331402629613876,
       "step": 280
     },
     {
       "epoch": 0.8774583963691377,
-      "grad_norm": 0.7166978012385395,
+      "grad_norm": 0.7014258527565755,
       "learning_rate": 8.818365836066101e-07,
-      "loss": 3.1621,
-      "mean_token_accuracy": 0.4014370396733284,
+      "loss": 3.1617,
+      "mean_token_accuracy": 0.40136476159095763,
       "step": 290
     },
     {
       "epoch": 0.9077155824508321,
-      "grad_norm": 0.671285323335886,
+      "grad_norm": 0.674864858075726,
       "learning_rate": 4.992888225905467e-07,
       "loss": 3.2242,
-      "mean_token_accuracy": 0.3944755882024765,
+      "mean_token_accuracy": 0.39469979852437975,
       "step": 300
     },
     {
       "epoch": 0.9077155824508321,
-      "eval_runtime": 0.2237,
-      "eval_samples_per_second": 214.554,
-      "eval_steps_per_second": 13.41,
+      "eval_runtime": 0.2285,
+      "eval_samples_per_second": 210.05,
+      "eval_steps_per_second": 13.128,
       "step": 300
     },
     {
       "epoch": 0.9379727685325264,
-      "grad_norm": 0.6982272552320608,
+      "grad_norm": 0.6759875022616182,
       "learning_rate": 2.2294435517691504e-07,
       "loss": 3.1918,
-      "mean_token_accuracy": 0.39713545739650724,
+      "mean_token_accuracy": 0.39749539494514463,
       "step": 310
     },
     {
       "epoch": 0.9682299546142209,
-      "grad_norm": 0.6635317905732051,
+      "grad_norm": 0.6680457304280091,
       "learning_rate": 5.5892286176932875e-08,
-      "loss": 3.1957,
-      "mean_token_accuracy": 0.3961677893996239,
+      "loss": 3.1953,
+      "mean_token_accuracy": 0.3963525503873825,
       "step": 320
     },
     {
       "epoch": 0.9984871406959153,
-      "grad_norm": 0.6569802422298082,
+      "grad_norm": 0.6551428033391504,
       "learning_rate": 0.0,
       "loss": 3.2023,
-      "mean_token_accuracy": 0.3949317991733551,
+      "mean_token_accuracy": 0.3949146747589111,
       "step": 330
     },
     {
       "epoch": 0.9984871406959153,
       "step": 330,
       "total_flos": 5419400756723712.0,
-      "train_loss": 3.61015625,
-      "train_runtime": 338.5176,
-      "train_samples_per_second": 31.227,
-      "train_steps_per_second": 0.975
+      "train_loss": 3.609872159090909,
+      "train_runtime": 919.7134,
+      "train_samples_per_second": 11.494,
+      "train_steps_per_second": 0.359
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c63eefca7f89a43cb29b332e3e763cfdc0b88b24fde057b72ff9133306fc667
+oid sha256:1189d0e125ab189d52bbe1c7c63f4aa7c75d5688e7d6e4ca9fc82551045c1eac
 size 7352