Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: openai-community/gpt2
-datasets: HuggingFaceFW/fineweb-edu
 library_name: transformers
 model_name: gpt2-RMT-2
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - sft
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for gpt2-RMT-2
 
-This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceFW/fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/1xooe0sm) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/mh05rq2b) 
 
 
 This model was trained with SFT.

all_results.json

@@ -3,10 +3,10 @@
     "eval_samples": 100,
     "eval_samples_per_second": 192.26,
     "eval_steps_per_second": 12.016,
-    "total_flos": 541940076511232.0,
-    "train_loss": 5.546875,
-    "train_runtime": 36.3584,
+    "total_flos": 5419400756723712.0,
+    "train_loss": 3.6152698863636363,
+    "train_runtime": 337.2618,
     "train_samples": 19883,
-    "train_samples_per_second": 29.074,
-    "train_steps_per_second": 0.908
+    "train_samples_per_second": 31.344,
+    "train_steps_per_second": 0.978
 }

config.json

@@ -33,6 +33,6 @@
   },
   "torch_dtype": "bfloat16",
   "transformers_version": "4.50.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 50257
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ef6b24bad9283be8a3270f3898cb6e1d27404fa045313443fde712995a1cabc
+oid sha256:2b16e3936aa2c91fc567c45311766ae7a01909050a6b9a4ce29225967ebbf4bc
 size 248912768

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 541940076511232.0,
-    "train_loss": 5.546875,
-    "train_runtime": 36.3584,
+    "total_flos": 5419400756723712.0,
+    "train_loss": 3.6152698863636363,
+    "train_runtime": 337.2618,
     "train_samples": 19883,
-    "train_samples_per_second": 29.074,
-    "train_steps_per_second": 0.908
+    "train_samples_per_second": 31.344,
+    "train_steps_per_second": 0.978
 }

trainer_state.json

@@ -1,50 +1,310 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.09984871406959153,
+  "epoch": 0.9984871406959153,
   "eval_steps": 100,
-  "global_step": 33,
+  "global_step": 330,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.030257186081694403,
-      "grad_norm": 18.3522235886847,
-      "learning_rate": 1.796093065705644e-05,
-      "loss": 6.8937,
-      "mean_token_accuracy": 0.1434693343937397,
+      "grad_norm": 51.5472786539882,
+      "learning_rate": 6.060606060606061e-06,
+      "loss": 7.3992,
+      "mean_token_accuracy": 0.13563802018761634,
       "step": 10
     },
     {
       "epoch": 0.060514372163388806,
-      "grad_norm": 10.614870433655174,
-      "learning_rate": 8.382180034472353e-06,
-      "loss": 5.3195,
-      "mean_token_accuracy": 0.1919048972427845,
+      "grad_norm": 27.69300301659005,
+      "learning_rate": 1.2121212121212122e-05,
+      "loss": 6.8242,
+      "mean_token_accuracy": 0.14449115768074988,
       "step": 20
     },
     {
       "epoch": 0.0907715582450832,
-      "grad_norm": 7.571339027683935,
-      "learning_rate": 5.234682881719766e-07,
-      "loss": 4.707,
-      "mean_token_accuracy": 0.23758598268032075,
+      "grad_norm": 9.743705978929961,
+      "learning_rate": 1.8181818181818182e-05,
+      "loss": 5.6266,
+      "mean_token_accuracy": 0.1745789811015129,
       "step": 30
     },
     {
-      "epoch": 0.09984871406959153,
-      "mean_token_accuracy": 0.24127203722794852,
-      "step": 33,
-      "total_flos": 541940076511232.0,
-      "train_loss": 5.546875,
-      "train_runtime": 36.3584,
-      "train_samples_per_second": 29.074,
-      "train_steps_per_second": 0.908
+      "epoch": 0.12102874432677761,
+      "grad_norm": 4.50812560927562,
+      "learning_rate": 1.9972599751485225e-05,
+      "loss": 4.6586,
+      "mean_token_accuracy": 0.24351915791630746,
+      "step": 40
+    },
+    {
+      "epoch": 0.15128593040847202,
+      "grad_norm": 3.1996689209062676,
+      "learning_rate": 1.9838755799290993e-05,
+      "loss": 4.0187,
+      "mean_token_accuracy": 0.3023824915289879,
+      "step": 50
+    },
+    {
+      "epoch": 0.1815431164901664,
+      "grad_norm": 1.8824084218377581,
+      "learning_rate": 1.9594929736144978e-05,
+      "loss": 3.6316,
+      "mean_token_accuracy": 0.3426647365093231,
+      "step": 60
+    },
+    {
+      "epoch": 0.2118003025718608,
+      "grad_norm": 1.1187763043308236,
+      "learning_rate": 1.9243847161266924e-05,
+      "loss": 3.3883,
+      "mean_token_accuracy": 0.3721240028738976,
+      "step": 70
+    },
+    {
+      "epoch": 0.24205748865355523,
+      "grad_norm": 1.0093619517882768,
+      "learning_rate": 1.8789432636206197e-05,
+      "loss": 3.3281,
+      "mean_token_accuracy": 0.3791012689471245,
+      "step": 80
+    },
+    {
+      "epoch": 0.2723146747352496,
+      "grad_norm": 0.8420416221184239,
+      "learning_rate": 1.8236765814298328e-05,
+      "loss": 3.2738,
+      "mean_token_accuracy": 0.3867510572075844,
+      "step": 90
+    },
+    {
+      "epoch": 0.30257186081694404,
+      "grad_norm": 0.8952752980575562,
+      "learning_rate": 1.7592024657977432e-05,
+      "loss": 3.2816,
+      "mean_token_accuracy": 0.3850349023938179,
+      "step": 100
+    },
+    {
+      "epoch": 0.30257186081694404,
+      "eval_runtime": 0.2249,
+      "eval_samples_per_second": 213.425,
+      "eval_steps_per_second": 13.339,
+      "step": 100
+    },
+    {
+      "epoch": 0.3328290468986384,
+      "grad_norm": 0.8146530137770093,
+      "learning_rate": 1.686241637868734e-05,
+      "loss": 3.2516,
+      "mean_token_accuracy": 0.3902030408382416,
+      "step": 110
+    },
+    {
+      "epoch": 0.3630862329803328,
+      "grad_norm": 0.7245362247896451,
+      "learning_rate": 1.6056096871376667e-05,
+      "loss": 3.2777,
+      "mean_token_accuracy": 0.3867401584982872,
+      "step": 120
+    },
+    {
+      "epoch": 0.39334341906202724,
+      "grad_norm": 0.7046871234836425,
+      "learning_rate": 1.5182079544175957e-05,
+      "loss": 3.25,
+      "mean_token_accuracy": 0.39042258113622663,
+      "step": 130
+    },
+    {
+      "epoch": 0.4236006051437216,
+      "grad_norm": 0.7889558903633703,
+      "learning_rate": 1.4250134562400301e-05,
+      "loss": 3.2254,
+      "mean_token_accuracy": 0.3929138645529747,
+      "step": 140
+    },
+    {
+      "epoch": 0.45385779122541603,
+      "grad_norm": 0.7526789075759445,
+      "learning_rate": 1.3270679633174219e-05,
+      "loss": 3.2168,
+      "mean_token_accuracy": 0.3945690140128136,
+      "step": 150
+    },
+    {
+      "epoch": 0.48411497730711045,
+      "grad_norm": 0.7037431923979744,
+      "learning_rate": 1.2254663551538047e-05,
+      "loss": 3.1898,
+      "mean_token_accuracy": 0.39688222110271454,
+      "step": 160
+    },
+    {
+      "epoch": 0.5143721633888049,
+      "grad_norm": 0.7269697505833069,
+      "learning_rate": 1.121344380981082e-05,
+      "loss": 3.2254,
+      "mean_token_accuracy": 0.39344793409109113,
+      "step": 170
+    },
+    {
+      "epoch": 0.5446293494704992,
+      "grad_norm": 0.71564273988106,
+      "learning_rate": 1.015865963834808e-05,
+      "loss": 3.198,
+      "mean_token_accuracy": 0.3971615001559258,
+      "step": 180
+    },
+    {
+      "epoch": 0.5748865355521936,
+      "grad_norm": 0.7272400173576803,
+      "learning_rate": 9.102101896903084e-06,
+      "loss": 3.2098,
+      "mean_token_accuracy": 0.3965013727545738,
+      "step": 190
+    },
+    {
+      "epoch": 0.6051437216338881,
+      "grad_norm": 0.7204839560593628,
+      "learning_rate": 8.055581271005292e-06,
+      "loss": 3.209,
+      "mean_token_accuracy": 0.394715978205204,
+      "step": 200
+    },
+    {
+      "epoch": 0.6051437216338881,
+      "eval_runtime": 0.2231,
+      "eval_samples_per_second": 215.144,
+      "eval_steps_per_second": 13.447,
+      "step": 200
+    },
+    {
+      "epoch": 0.6354009077155824,
+      "grad_norm": 0.6620216126492257,
+      "learning_rate": 7.0307962467172555e-06,
+      "loss": 3.1988,
+      "mean_token_accuracy": 0.39781390130519867,
+      "step": 210
+    },
+    {
+      "epoch": 0.6656580937972768,
+      "grad_norm": 0.7813459496424706,
+      "learning_rate": 6.039202339608432e-06,
+      "loss": 3.202,
+      "mean_token_accuracy": 0.3953568756580353,
+      "step": 220
+    },
+    {
+      "epoch": 0.6959152798789713,
+      "grad_norm": 0.6556046091666947,
+      "learning_rate": 5.091884039764321e-06,
+      "loss": 3.1969,
+      "mean_token_accuracy": 0.3961416259407997,
+      "step": 230
+    },
+    {
+      "epoch": 0.7261724659606656,
+      "grad_norm": 0.7243524981690901,
+      "learning_rate": 4.19943090428802e-06,
+      "loss": 3.2102,
+      "mean_token_accuracy": 0.3935989648103714,
+      "step": 240
+    },
+    {
+      "epoch": 0.75642965204236,
+      "grad_norm": 0.7423682208710032,
+      "learning_rate": 3.37181918238904e-06,
+      "loss": 3.2078,
+      "mean_token_accuracy": 0.39539735913276675,
+      "step": 250
+    },
+    {
+      "epoch": 0.7866868381240545,
+      "grad_norm": 0.684794452564992,
+      "learning_rate": 2.618300296308135e-06,
+      "loss": 3.2219,
+      "mean_token_accuracy": 0.3939197212457657,
+      "step": 260
+    },
+    {
+      "epoch": 0.8169440242057489,
+      "grad_norm": 0.6857522826014064,
+      "learning_rate": 1.947297424689414e-06,
+      "loss": 3.1992,
+      "mean_token_accuracy": 0.39573368430137634,
+      "step": 270
+    },
+    {
+      "epoch": 0.8472012102874432,
+      "grad_norm": 0.6966190236263456,
+      "learning_rate": 1.3663113444380905e-06,
+      "loss": 3.2031,
+      "mean_token_accuracy": 0.3933654069900513,
+      "step": 280
+    },
+    {
+      "epoch": 0.8774583963691377,
+      "grad_norm": 0.8226214667450941,
+      "learning_rate": 8.818365836066101e-07,
+      "loss": 3.1621,
+      "mean_token_accuracy": 0.4012774467468262,
+      "step": 290
+    },
+    {
+      "epoch": 0.9077155824508321,
+      "grad_norm": 0.6811554548251312,
+      "learning_rate": 4.992888225905467e-07,
+      "loss": 3.2254,
+      "mean_token_accuracy": 0.39436970353126527,
+      "step": 300
+    },
+    {
+      "epoch": 0.9077155824508321,
+      "eval_runtime": 0.223,
+      "eval_samples_per_second": 215.211,
+      "eval_steps_per_second": 13.451,
+      "step": 300
+    },
+    {
+      "epoch": 0.9379727685325264,
+      "grad_norm": 0.6793742171946109,
+      "learning_rate": 2.2294435517691504e-07,
+      "loss": 3.1926,
+      "mean_token_accuracy": 0.397060464322567,
+      "step": 310
+    },
+    {
+      "epoch": 0.9682299546142209,
+      "grad_norm": 0.6744479467043031,
+      "learning_rate": 5.5892286176932875e-08,
+      "loss": 3.1961,
+      "mean_token_accuracy": 0.3960541605949402,
+      "step": 320
+    },
+    {
+      "epoch": 0.9984871406959153,
+      "grad_norm": 0.6621754518709608,
+      "learning_rate": 0.0,
+      "loss": 3.2035,
+      "mean_token_accuracy": 0.3947854354977608,
+      "step": 330
+    },
+    {
+      "epoch": 0.9984871406959153,
+      "step": 330,
+      "total_flos": 5419400756723712.0,
+      "train_loss": 3.6152698863636363,
+      "train_runtime": 337.2618,
+      "train_samples_per_second": 31.344,
+      "train_steps_per_second": 0.978
     }
   ],
   "logging_steps": 10,
-  "max_steps": 33,
+  "max_steps": 330,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -60,7 +320,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 541940076511232.0,
+  "total_flos": 5419400756723712.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:baf06d605ca2ac794fb0cb09d28d4e25826d2c1b68fdaaefca458cb0a917d847
+oid sha256:0dba648ef7d2addc48fb53ca1ab6b041442778aef087f173aef4c1a8ab1b68d9
 size 7352