Model save

README.md

@@ -1,19 +1,15 @@
 ---
 base_model: openai-community/gpt2
-datasets: HuggingFaceFW/fineweb-edu
 library_name: transformers
 model_name: gpt2-RMT-2
 tags:
 - generated_from_trainer
-- open-r1
-- trl
-- sft
 licence: license
 ---
 
 # Model Card for gpt2-RMT-2
 
-This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceFW/fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +25,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/xxpt9481) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/mbk8mzkf) 
 
 
 This model was trained with SFT.

all_results.json

@@ -3,10 +3,10 @@
     "eval_samples": 100,
     "eval_samples_per_second": 197.801,
     "eval_steps_per_second": 12.363,
-    "total_flos": 5419400756723712.0,
-    "train_loss": 3.609872159090909,
-    "train_runtime": 919.7134,
-    "train_samples": 19883,
-    "train_samples_per_second": 11.494,
-    "train_steps_per_second": 0.359
+    "total_flos": 0.0,
+    "train_loss": 3.0202497909698995,
+    "train_runtime": 60.0371,
+    "train_samples": 16610,
+    "train_samples_per_second": 79.507,
+    "train_steps_per_second": 4.98
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:585da17beba27ab616789ca4c8a12e9173b873bedfa95a97b6ec57e2b190642b
-size 248912768
+oid sha256:0b1cef2e205503013144a7bcad7ad852c288d9e54cc44a8aaf644d9d71ec6cbc
+size 326110968

tokenizer_config.json

@@ -11,6 +11,7 @@
     }
   },
   "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}{{ message['role'] }}: {{ message['content'] }}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 5419400756723712.0,
-    "train_loss": 3.609872159090909,
-    "train_runtime": 919.7134,
-    "train_samples": 19883,
-    "train_samples_per_second": 11.494,
-    "train_steps_per_second": 0.359
+    "total_flos": 0.0,
+    "train_loss": 3.0202497909698995,
+    "train_runtime": 60.0371,
+    "train_samples": 16610,
+    "train_samples_per_second": 79.507,
+    "train_steps_per_second": 4.98
 }

trainer_state.json

@@ -1,310 +1,272 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9984871406959153,
+  "epoch": 0.05010893246187364,
   "eval_steps": 100,
-  "global_step": 330,
+  "global_step": 299,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.030257186081694403,
-      "grad_norm": 52.88699541505755,
-      "learning_rate": 6.060606060606061e-06,
-      "loss": 7.4,
-      "mean_token_accuracy": 0.13546177372336388,
+      "epoch": 0.0016758840288252053,
+      "grad_norm": 36.404178619384766,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 5.6828,
+      "mean_token_accuracy": 0.27888479083776474,
       "step": 10
     },
     {
-      "epoch": 0.060514372163388806,
-      "grad_norm": 26.843614632955404,
-      "learning_rate": 1.2121212121212122e-05,
-      "loss": 6.8156,
-      "mean_token_accuracy": 0.14476052597165107,
+      "epoch": 0.0033517680576504107,
+      "grad_norm": 42.79513931274414,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 5.425,
+      "mean_token_accuracy": 0.2772253334522247,
       "step": 20
     },
     {
-      "epoch": 0.0907715582450832,
-      "grad_norm": 9.852827567927251,
-      "learning_rate": 1.8181818181818182e-05,
-      "loss": 5.6055,
-      "mean_token_accuracy": 0.17639728486537934,
+      "epoch": 0.005027652086475616,
+      "grad_norm": 17.52666664123535,
+      "learning_rate": 2e-05,
+      "loss": 4.6359,
+      "mean_token_accuracy": 0.2905010163784027,
       "step": 30
     },
     {
-      "epoch": 0.12102874432677761,
-      "grad_norm": 4.231145666737617,
-      "learning_rate": 1.9972599751485225e-05,
-      "loss": 4.6,
-      "mean_token_accuracy": 0.2488641396164894,
+      "epoch": 0.006703536115300821,
+      "grad_norm": 39.22002410888672,
+      "learning_rate": 1.9931880491250263e-05,
+      "loss": 4.1836,
+      "mean_token_accuracy": 0.29999364018440244,
       "step": 40
     },
     {
-      "epoch": 0.15128593040847202,
-      "grad_norm": 3.2271685140325594,
-      "learning_rate": 1.9838755799290993e-05,
-      "loss": 3.9941,
-      "mean_token_accuracy": 0.3051066160202026,
+      "epoch": 0.008379420144126026,
+      "grad_norm": 6.146772861480713,
+      "learning_rate": 1.9728450018495506e-05,
+      "loss": 3.9766,
+      "mean_token_accuracy": 0.30167217552661896,
       "step": 50
     },
     {
-      "epoch": 0.1815431164901664,
-      "grad_norm": 1.885693756577763,
-      "learning_rate": 1.9594929736144978e-05,
-      "loss": 3.609,
-      "mean_token_accuracy": 0.34526345133781433,
+      "epoch": 0.010055304172951232,
+      "grad_norm": 6.285459995269775,
+      "learning_rate": 1.9392480098509488e-05,
+      "loss": 3.7109,
+      "mean_token_accuracy": 0.32380467653274536,
       "step": 60
     },
     {
-      "epoch": 0.2118003025718608,
-      "grad_norm": 1.1244069323463834,
-      "learning_rate": 1.9243847161266924e-05,
-      "loss": 3.3801,
-      "mean_token_accuracy": 0.37311544418334963,
+      "epoch": 0.011731188201776438,
+      "grad_norm": 5.058553218841553,
+      "learning_rate": 1.8928547952473037e-05,
+      "loss": 3.5086,
+      "mean_token_accuracy": 0.345212359726429,
       "step": 70
     },
     {
-      "epoch": 0.24205748865355523,
-      "grad_norm": 0.9867146004226443,
-      "learning_rate": 1.8789432636206197e-05,
-      "loss": 3.3258,
-      "mean_token_accuracy": 0.3792398452758789,
+      "epoch": 0.013407072230601643,
+      "grad_norm": 15.02955150604248,
+      "learning_rate": 1.8342974146362397e-05,
+      "loss": 3.3859,
+      "mean_token_accuracy": 0.3557349964976311,
       "step": 80
     },
     {
-      "epoch": 0.2723146747352496,
-      "grad_norm": 0.8107915407779778,
-      "learning_rate": 1.8236765814298328e-05,
-      "loss": 3.2719,
-      "mean_token_accuracy": 0.38700797259807584,
+      "epoch": 0.015082956259426848,
+      "grad_norm": 4.372200965881348,
+      "learning_rate": 1.7643736480179353e-05,
+      "loss": 3.2352,
+      "mean_token_accuracy": 0.3736902341246605,
       "step": 90
     },
     {
-      "epoch": 0.30257186081694404,
-      "grad_norm": 0.8941659318929186,
-      "learning_rate": 1.7592024657977432e-05,
-      "loss": 3.2785,
-      "mean_token_accuracy": 0.38543802350759504,
+      "epoch": 0.01675884028825205,
+      "grad_norm": 3.283221960067749,
+      "learning_rate": 1.684036129918786e-05,
+      "loss": 3.0875,
+      "mean_token_accuracy": 0.3915310278534889,
       "step": 100
     },
     {
-      "epoch": 0.30257186081694404,
-      "eval_runtime": 0.237,
-      "eval_samples_per_second": 202.542,
-      "eval_steps_per_second": 12.659,
+      "epoch": 0.01675884028825205,
+      "eval_runtime": 2.7416,
+      "eval_samples_per_second": 208.276,
+      "eval_steps_per_second": 52.16,
       "step": 100
     },
     {
-      "epoch": 0.3328290468986384,
-      "grad_norm": 0.7913271033960183,
-      "learning_rate": 1.686241637868734e-05,
-      "loss": 3.25,
-      "mean_token_accuracy": 0.3904023408889771,
+      "epoch": 0.01843472431707726,
+      "grad_norm": 17.207765579223633,
+      "learning_rate": 1.5943793707922086e-05,
+      "loss": 2.9906,
+      "mean_token_accuracy": 0.40075025260448455,
       "step": 110
     },
     {
-      "epoch": 0.3630862329803328,
-      "grad_norm": 0.7385462564687278,
-      "learning_rate": 1.6056096871376667e-05,
-      "loss": 3.2746,
-      "mean_token_accuracy": 0.3869674876332283,
+      "epoch": 0.020110608345902465,
+      "grad_norm": 3.305183172225952,
+      "learning_rate": 1.4966248455157622e-05,
+      "loss": 2.8484,
+      "mean_token_accuracy": 0.42432604134082796,
       "step": 120
     },
     {
-      "epoch": 0.39334341906202724,
-      "grad_norm": 0.7171108615447848,
-      "learning_rate": 1.5182079544175957e-05,
-      "loss": 3.2488,
-      "mean_token_accuracy": 0.39062655717134476,
+      "epoch": 0.02178649237472767,
+      "grad_norm": 3.639101266860962,
+      "learning_rate": 1.392104352137426e-05,
+      "loss": 2.7719,
+      "mean_token_accuracy": 0.4330811306834221,
       "step": 130
     },
     {
-      "epoch": 0.4236006051437216,
-      "grad_norm": 0.7917794437598532,
-      "learning_rate": 1.4250134562400301e-05,
-      "loss": 3.2227,
-      "mean_token_accuracy": 0.39324862957000734,
+      "epoch": 0.023462376403552875,
+      "grad_norm": 2.6281790733337402,
+      "learning_rate": 1.2822418675898428e-05,
+      "loss": 2.6055,
+      "mean_token_accuracy": 0.46078331768512726,
       "step": 140
     },
     {
-      "epoch": 0.45385779122541603,
-      "grad_norm": 0.7618905726241425,
-      "learning_rate": 1.3270679633174219e-05,
-      "loss": 3.2133,
-      "mean_token_accuracy": 0.39474184066057205,
+      "epoch": 0.02513826043237808,
+      "grad_norm": 2.1558072566986084,
+      "learning_rate": 1.1685341475684935e-05,
+      "loss": 2.5195,
+      "mean_token_accuracy": 0.4750635787844658,
       "step": 150
     },
     {
-      "epoch": 0.48411497730711045,
-      "grad_norm": 0.710429734736302,
-      "learning_rate": 1.2254663551538047e-05,
-      "loss": 3.1895,
-      "mean_token_accuracy": 0.3969797551631927,
+      "epoch": 0.026814144461203285,
+      "grad_norm": 3.083050489425659,
+      "learning_rate": 1.0525303348791599e-05,
+      "loss": 2.5078,
+      "mean_token_accuracy": 0.4781091034412384,
       "step": 160
     },
     {
-      "epoch": 0.5143721633888049,
-      "grad_norm": 0.7314245404422203,
-      "learning_rate": 1.121344380981082e-05,
-      "loss": 3.2254,
-      "mean_token_accuracy": 0.39376401603221894,
+      "epoch": 0.02849002849002849,
+      "grad_norm": 1.9051162004470825,
+      "learning_rate": 9.358108540685406e-06,
+      "loss": 2.4273,
+      "mean_token_accuracy": 0.4916709065437317,
       "step": 170
     },
     {
-      "epoch": 0.5446293494704992,
-      "grad_norm": 0.711095968768027,
-      "learning_rate": 1.015865963834808e-05,
-      "loss": 3.1969,
-      "mean_token_accuracy": 0.39729384928941724,
+      "epoch": 0.030165912518853696,
+      "grad_norm": 1.8084359169006348,
+      "learning_rate": 8.199658798755048e-06,
+      "loss": 2.4664,
+      "mean_token_accuracy": 0.4778802111744881,
       "step": 180
     },
     {
-      "epoch": 0.5748865355521936,
-      "grad_norm": 0.7048692336249331,
-      "learning_rate": 9.102101896903084e-06,
-      "loss": 3.2078,
-      "mean_token_accuracy": 0.3967027798295021,
+      "epoch": 0.0318417965476789,
+      "grad_norm": 1.7160115242004395,
+      "learning_rate": 7.065736728466832e-06,
+      "loss": 2.3969,
+      "mean_token_accuracy": 0.49128942042589185,
       "step": 190
     },
     {
-      "epoch": 0.6051437216338881,
-      "grad_norm": 0.714339105324786,
-      "learning_rate": 8.055581271005292e-06,
-      "loss": 3.2086,
-      "mean_token_accuracy": 0.3946896642446518,
+      "epoch": 0.0335176805765041,
+      "grad_norm": 1.6504725217819214,
+      "learning_rate": 5.971790772698467e-06,
+      "loss": 2.4383,
+      "mean_token_accuracy": 0.4894074246287346,
       "step": 200
     },
     {
-      "epoch": 0.6051437216338881,
-      "eval_runtime": 0.2292,
-      "eval_samples_per_second": 209.454,
-      "eval_steps_per_second": 13.091,
+      "epoch": 0.0335176805765041,
+      "eval_runtime": 2.7088,
+      "eval_samples_per_second": 210.793,
+      "eval_steps_per_second": 52.79,
       "step": 200
     },
     {
-      "epoch": 0.6354009077155824,
-      "grad_norm": 0.732439534421158,
-      "learning_rate": 7.0307962467172555e-06,
-      "loss": 3.198,
-      "mean_token_accuracy": 0.39805368781089784,
+      "epoch": 0.03519356460532931,
+      "grad_norm": 1.851637363433838,
+      "learning_rate": 4.932724743671089e-06,
+      "loss": 2.3555,
+      "mean_token_accuracy": 0.5041772618889808,
       "step": 210
     },
     {
-      "epoch": 0.6656580937972768,
-      "grad_norm": 0.7679520615376684,
-      "learning_rate": 6.039202339608432e-06,
-      "loss": 3.2008,
-      "mean_token_accuracy": 0.39553126841783526,
+      "epoch": 0.03686944863415452,
+      "grad_norm": 1.8771989345550537,
+      "learning_rate": 3.96269477487588e-06,
+      "loss": 2.3977,
+      "mean_token_accuracy": 0.4926373302936554,
       "step": 220
     },
     {
-      "epoch": 0.6959152798789713,
-      "grad_norm": 0.6598246730886332,
-      "learning_rate": 5.091884039764321e-06,
-      "loss": 3.1961,
-      "mean_token_accuracy": 0.3962506249547005,
+      "epoch": 0.03854533266297972,
+      "grad_norm": 4.372796535491943,
+      "learning_rate": 3.074916459302211e-06,
+      "loss": 2.3508,
+      "mean_token_accuracy": 0.5035096645355225,
       "step": 230
     },
     {
-      "epoch": 0.7261724659606656,
-      "grad_norm": 0.6582768941498197,
-      "learning_rate": 4.19943090428802e-06,
-      "loss": 3.2105,
-      "mean_token_accuracy": 0.3936332181096077,
+      "epoch": 0.04022121669180493,
+      "grad_norm": 1.8549838066101074,
+      "learning_rate": 2.281484801497186e-06,
+      "loss": 2.4258,
+      "mean_token_accuracy": 0.48886062651872636,
       "step": 240
     },
     {
-      "epoch": 0.75642965204236,
-      "grad_norm": 0.7382830086371551,
-      "learning_rate": 3.37181918238904e-06,
-      "loss": 3.207,
-      "mean_token_accuracy": 0.39551725089550016,
+      "epoch": 0.04189710072063013,
+      "grad_norm": 1.7765941619873047,
+      "learning_rate": 1.5932094364120453e-06,
+      "loss": 2.3656,
+      "mean_token_accuracy": 0.4967192322015762,
       "step": 250
     },
     {
-      "epoch": 0.7866868381240545,
-      "grad_norm": 0.6780081878254819,
-      "learning_rate": 2.618300296308135e-06,
-      "loss": 3.2203,
-      "mean_token_accuracy": 0.393978887796402,
+      "epoch": 0.04357298474945534,
+      "grad_norm": 1.8978796005249023,
+      "learning_rate": 1.0194673599976134e-06,
+      "loss": 2.3398,
+      "mean_token_accuracy": 0.5044379487633706,
       "step": 260
     },
     {
-      "epoch": 0.8169440242057489,
-      "grad_norm": 0.6887657468672304,
-      "learning_rate": 1.947297424689414e-06,
-      "loss": 3.1984,
-      "mean_token_accuracy": 0.39577416330575943,
+      "epoch": 0.04524886877828054,
+      "grad_norm": 2.0636632442474365,
+      "learning_rate": 5.680751779327742e-07,
+      "loss": 2.3687,
+      "mean_token_accuracy": 0.49782553166151045,
       "step": 270
     },
     {
-      "epoch": 0.8472012102874432,
-      "grad_norm": 0.6760254781738906,
-      "learning_rate": 1.3663113444380905e-06,
-      "loss": 3.2012,
-      "mean_token_accuracy": 0.39331402629613876,
+      "epoch": 0.04692475280710575,
+      "grad_norm": 1.694811224937439,
+      "learning_rate": 2.4518261295667255e-07,
+      "loss": 2.3867,
+      "mean_token_accuracy": 0.4927390620112419,
       "step": 280
     },
     {
-      "epoch": 0.8774583963691377,
-      "grad_norm": 0.7014258527565755,
-      "learning_rate": 8.818365836066101e-07,
-      "loss": 3.1617,
-      "mean_token_accuracy": 0.40136476159095763,
+      "epoch": 0.04860063683593095,
+      "grad_norm": 2.2234456539154053,
+      "learning_rate": 5.518872165033329e-08,
+      "loss": 2.3953,
+      "mean_token_accuracy": 0.49295524060726165,
       "step": 290
     },
     {
-      "epoch": 0.9077155824508321,
-      "grad_norm": 0.674864858075726,
-      "learning_rate": 4.992888225905467e-07,
-      "loss": 3.2242,
-      "mean_token_accuracy": 0.39469979852437975,
-      "step": 300
-    },
-    {
-      "epoch": 0.9077155824508321,
-      "eval_runtime": 0.2285,
-      "eval_samples_per_second": 210.05,
-      "eval_steps_per_second": 13.128,
-      "step": 300
-    },
-    {
-      "epoch": 0.9379727685325264,
-      "grad_norm": 0.6759875022616182,
-      "learning_rate": 2.2294435517691504e-07,
-      "loss": 3.1918,
-      "mean_token_accuracy": 0.39749539494514463,
-      "step": 310
-    },
-    {
-      "epoch": 0.9682299546142209,
-      "grad_norm": 0.6680457304280091,
-      "learning_rate": 5.5892286176932875e-08,
-      "loss": 3.1953,
-      "mean_token_accuracy": 0.3963525503873825,
-      "step": 320
-    },
-    {
-      "epoch": 0.9984871406959153,
-      "grad_norm": 0.6551428033391504,
-      "learning_rate": 0.0,
-      "loss": 3.2023,
-      "mean_token_accuracy": 0.3949146747589111,
-      "step": 330
-    },
-    {
-      "epoch": 0.9984871406959153,
-      "step": 330,
-      "total_flos": 5419400756723712.0,
-      "train_loss": 3.609872159090909,
-      "train_runtime": 919.7134,
-      "train_samples_per_second": 11.494,
-      "train_steps_per_second": 0.359
+      "epoch": 0.05010893246187364,
+      "mean_token_accuracy": 0.4976404458284378,
+      "step": 299,
+      "total_flos": 0.0,
+      "train_loss": 3.0202497909698995,
+      "train_runtime": 60.0371,
+      "train_samples_per_second": 79.507,
+      "train_steps_per_second": 4.98
     }
   ],
   "logging_steps": 10,
-  "max_steps": 330,
+  "max_steps": 299,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -320,7 +282,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 5419400756723712.0,
+  "total_flos": 0.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1189d0e125ab189d52bbe1c7c63f4aa7c75d5688e7d6e4ca9fc82551045c1eac
+oid sha256:bdd0e83665f82cbff34945a1dc5f700141602e90d62707ec1ecd8225c96d8e93
 size 7352