End of training

README.md

@@ -1,6 +1,6 @@
 ---
 base_model: openai-community/gpt2
-datasets: HuggingFaceH4/Bespoke-Stratos-17k
+datasets: HuggingFaceFW/fineweb-edu
 library_name: transformers
 model_name: gpt2-RMT-2
 tags:
@@ -11,7 +11,7 @@ licence: license
 
 # Model Card for gpt2-RMT-2
 
-This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceH4/Bespoke-Stratos-17k](https://huggingface.co/datasets/HuggingFaceH4/Bespoke-Stratos-17k) dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceFW/fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) dataset.
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/mbk8mzkf) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/t9renl1y) 
 
 
 This model was trained with SFT.

all_results.json

@@ -1,12 +1,12 @@
 {
-    "eval_runtime": 2.7326,
+    "eval_runtime": 0.3164,
     "eval_samples": 100,
-    "eval_samples_per_second": 208.955,
-    "eval_steps_per_second": 52.33,
+    "eval_samples_per_second": 151.686,
+    "eval_steps_per_second": 9.48,
     "total_flos": 0.0,
-    "train_loss": 3.0202497909698995,
-    "train_runtime": 60.0371,
-    "train_samples": 16610,
-    "train_samples_per_second": 79.507,
-    "train_steps_per_second": 4.98
+    "train_loss": 5.556107954545454,
+    "train_runtime": 94.197,
+    "train_samples": 19883,
+    "train_samples_per_second": 11.222,
+    "train_steps_per_second": 0.35
 }

base_model/config.json

@@ -0,0 +1,38 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0.dev0",
+  "use_cache": false,
+  "vocab_size": 50257
+}

base_model/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.50.0.dev0"
+}

base_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d237600614a23ab38e6e8626c1a3495f6abbc5bc35856fee8d2c293889dbd3f
+size 248912768

config.json

@@ -34,6 +34,6 @@
   },
   "torch_dtype": "bfloat16",
   "transformers_version": "4.50.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 50257
 }

eval_results.json

@@ -1,6 +1,6 @@
 {
-    "eval_runtime": 2.7326,
+    "eval_runtime": 0.3164,
     "eval_samples": 100,
-    "eval_samples_per_second": 208.955,
-    "eval_steps_per_second": 52.33
+    "eval_samples_per_second": 151.686,
+    "eval_steps_per_second": 9.48
 }

generation_config.json

@@ -1,4 +1,6 @@
 {
   "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
   "transformers_version": "4.50.0.dev0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b1cef2e205503013144a7bcad7ad852c288d9e54cc44a8aaf644d9d71ec6cbc
-size 326110968
+oid sha256:ddafb83cf1a59157af80d355275bd09239a37d622ab84d9c4df9fd1bb357cbce
+size 326089656

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e07ea7caa7f567049c6ac949b18c3c222e52a15c44b05ed9bc048bb7eda52472
+oid sha256:b8e45365da2e62bd9b60b592a1fdd35a62ba9baf557b86a6caafeefd31fc75d5
 size 248944286

tokenizer_config.json

@@ -11,7 +11,6 @@
     }
   },
   "bos_token": "<|endoftext|>",
-  "chat_template": "{% for message in messages %}{{ message['role'] }}: {{ message['content'] }}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.0202497909698995,
-    "train_runtime": 60.0371,
-    "train_samples": 16610,
-    "train_samples_per_second": 79.507,
-    "train_steps_per_second": 4.98
+    "train_loss": 5.556107954545454,
+    "train_runtime": 94.197,
+    "train_samples": 19883,
+    "train_samples_per_second": 11.222,
+    "train_steps_per_second": 0.35
 }

trainer_state.json

@@ -1,272 +1,50 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.05010893246187364,
+  "epoch": 0.09984871406959153,
   "eval_steps": 100,
-  "global_step": 299,
+  "global_step": 33,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0016758840288252053,
-      "grad_norm": 36.404178619384766,
-      "learning_rate": 6.666666666666667e-06,
-      "loss": 5.6828,
-      "mean_token_accuracy": 0.27888479083776474,
+      "epoch": 0.030257186081694403,
+      "grad_norm": 18.575438958798006,
+      "learning_rate": 1.796093065705644e-05,
+      "loss": 6.9031,
+      "mean_token_accuracy": 0.1433999717235565,
       "step": 10
     },
     {
-      "epoch": 0.0033517680576504107,
-      "grad_norm": 42.79513931274414,
-      "learning_rate": 1.3333333333333333e-05,
-      "loss": 5.425,
-      "mean_token_accuracy": 0.2772253334522247,
+      "epoch": 0.060514372163388806,
+      "grad_norm": 11.22312925711848,
+      "learning_rate": 8.382180034472353e-06,
+      "loss": 5.325,
+      "mean_token_accuracy": 0.19182548448443412,
       "step": 20
     },
     {
-      "epoch": 0.005027652086475616,
-      "grad_norm": 17.52666664123535,
-      "learning_rate": 2e-05,
-      "loss": 4.6359,
-      "mean_token_accuracy": 0.2905010163784027,
+      "epoch": 0.0907715582450832,
+      "grad_norm": 7.8614989055585625,
+      "learning_rate": 5.234682881719766e-07,
+      "loss": 4.7195,
+      "mean_token_accuracy": 0.23640244454145432,
       "step": 30
     },
     {
-      "epoch": 0.006703536115300821,
-      "grad_norm": 39.22002410888672,
-      "learning_rate": 1.9931880491250263e-05,
-      "loss": 4.1836,
-      "mean_token_accuracy": 0.29999364018440244,
-      "step": 40
-    },
-    {
-      "epoch": 0.008379420144126026,
-      "grad_norm": 6.146772861480713,
-      "learning_rate": 1.9728450018495506e-05,
-      "loss": 3.9766,
-      "mean_token_accuracy": 0.30167217552661896,
-      "step": 50
-    },
-    {
-      "epoch": 0.010055304172951232,
-      "grad_norm": 6.285459995269775,
-      "learning_rate": 1.9392480098509488e-05,
-      "loss": 3.7109,
-      "mean_token_accuracy": 0.32380467653274536,
-      "step": 60
-    },
-    {
-      "epoch": 0.011731188201776438,
-      "grad_norm": 5.058553218841553,
-      "learning_rate": 1.8928547952473037e-05,
-      "loss": 3.5086,
-      "mean_token_accuracy": 0.345212359726429,
-      "step": 70
-    },
-    {
-      "epoch": 0.013407072230601643,
-      "grad_norm": 15.02955150604248,
-      "learning_rate": 1.8342974146362397e-05,
-      "loss": 3.3859,
-      "mean_token_accuracy": 0.3557349964976311,
-      "step": 80
-    },
-    {
-      "epoch": 0.015082956259426848,
-      "grad_norm": 4.372200965881348,
-      "learning_rate": 1.7643736480179353e-05,
-      "loss": 3.2352,
-      "mean_token_accuracy": 0.3736902341246605,
-      "step": 90
-    },
-    {
-      "epoch": 0.01675884028825205,
-      "grad_norm": 3.283221960067749,
-      "learning_rate": 1.684036129918786e-05,
-      "loss": 3.0875,
-      "mean_token_accuracy": 0.3915310278534889,
-      "step": 100
-    },
-    {
-      "epoch": 0.01675884028825205,
-      "eval_runtime": 2.7416,
-      "eval_samples_per_second": 208.276,
-      "eval_steps_per_second": 52.16,
-      "step": 100
-    },
-    {
-      "epoch": 0.01843472431707726,
-      "grad_norm": 17.207765579223633,
-      "learning_rate": 1.5943793707922086e-05,
-      "loss": 2.9906,
-      "mean_token_accuracy": 0.40075025260448455,
-      "step": 110
-    },
-    {
-      "epoch": 0.020110608345902465,
-      "grad_norm": 3.305183172225952,
-      "learning_rate": 1.4966248455157622e-05,
-      "loss": 2.8484,
-      "mean_token_accuracy": 0.42432604134082796,
-      "step": 120
-    },
-    {
-      "epoch": 0.02178649237472767,
-      "grad_norm": 3.639101266860962,
-      "learning_rate": 1.392104352137426e-05,
-      "loss": 2.7719,
-      "mean_token_accuracy": 0.4330811306834221,
-      "step": 130
-    },
-    {
-      "epoch": 0.023462376403552875,
-      "grad_norm": 2.6281790733337402,
-      "learning_rate": 1.2822418675898428e-05,
-      "loss": 2.6055,
-      "mean_token_accuracy": 0.46078331768512726,
-      "step": 140
-    },
-    {
-      "epoch": 0.02513826043237808,
-      "grad_norm": 2.1558072566986084,
-      "learning_rate": 1.1685341475684935e-05,
-      "loss": 2.5195,
-      "mean_token_accuracy": 0.4750635787844658,
-      "step": 150
-    },
-    {
-      "epoch": 0.026814144461203285,
-      "grad_norm": 3.083050489425659,
-      "learning_rate": 1.0525303348791599e-05,
-      "loss": 2.5078,
-      "mean_token_accuracy": 0.4781091034412384,
-      "step": 160
-    },
-    {
-      "epoch": 0.02849002849002849,
-      "grad_norm": 1.9051162004470825,
-      "learning_rate": 9.358108540685406e-06,
-      "loss": 2.4273,
-      "mean_token_accuracy": 0.4916709065437317,
-      "step": 170
-    },
-    {
-      "epoch": 0.030165912518853696,
-      "grad_norm": 1.8084359169006348,
-      "learning_rate": 8.199658798755048e-06,
-      "loss": 2.4664,
-      "mean_token_accuracy": 0.4778802111744881,
-      "step": 180
-    },
-    {
-      "epoch": 0.0318417965476789,
-      "grad_norm": 1.7160115242004395,
-      "learning_rate": 7.065736728466832e-06,
-      "loss": 2.3969,
-      "mean_token_accuracy": 0.49128942042589185,
-      "step": 190
-    },
-    {
-      "epoch": 0.0335176805765041,
-      "grad_norm": 1.6504725217819214,
-      "learning_rate": 5.971790772698467e-06,
-      "loss": 2.4383,
-      "mean_token_accuracy": 0.4894074246287346,
-      "step": 200
-    },
-    {
-      "epoch": 0.0335176805765041,
-      "eval_runtime": 2.7088,
-      "eval_samples_per_second": 210.793,
-      "eval_steps_per_second": 52.79,
-      "step": 200
-    },
-    {
-      "epoch": 0.03519356460532931,
-      "grad_norm": 1.851637363433838,
-      "learning_rate": 4.932724743671089e-06,
-      "loss": 2.3555,
-      "mean_token_accuracy": 0.5041772618889808,
-      "step": 210
-    },
-    {
-      "epoch": 0.03686944863415452,
-      "grad_norm": 1.8771989345550537,
-      "learning_rate": 3.96269477487588e-06,
-      "loss": 2.3977,
-      "mean_token_accuracy": 0.4926373302936554,
-      "step": 220
-    },
-    {
-      "epoch": 0.03854533266297972,
-      "grad_norm": 4.372796535491943,
-      "learning_rate": 3.074916459302211e-06,
-      "loss": 2.3508,
-      "mean_token_accuracy": 0.5035096645355225,
-      "step": 230
-    },
-    {
-      "epoch": 0.04022121669180493,
-      "grad_norm": 1.8549838066101074,
-      "learning_rate": 2.281484801497186e-06,
-      "loss": 2.4258,
-      "mean_token_accuracy": 0.48886062651872636,
-      "step": 240
-    },
-    {
-      "epoch": 0.04189710072063013,
-      "grad_norm": 1.7765941619873047,
-      "learning_rate": 1.5932094364120453e-06,
-      "loss": 2.3656,
-      "mean_token_accuracy": 0.4967192322015762,
-      "step": 250
-    },
-    {
-      "epoch": 0.04357298474945534,
-      "grad_norm": 1.8978796005249023,
-      "learning_rate": 1.0194673599976134e-06,
-      "loss": 2.3398,
-      "mean_token_accuracy": 0.5044379487633706,
-      "step": 260
-    },
-    {
-      "epoch": 0.04524886877828054,
-      "grad_norm": 2.0636632442474365,
-      "learning_rate": 5.680751779327742e-07,
-      "loss": 2.3687,
-      "mean_token_accuracy": 0.49782553166151045,
-      "step": 270
-    },
-    {
-      "epoch": 0.04692475280710575,
-      "grad_norm": 1.694811224937439,
-      "learning_rate": 2.4518261295667255e-07,
-      "loss": 2.3867,
-      "mean_token_accuracy": 0.4927390620112419,
-      "step": 280
-    },
-    {
-      "epoch": 0.04860063683593095,
-      "grad_norm": 2.2234456539154053,
-      "learning_rate": 5.518872165033329e-08,
-      "loss": 2.3953,
-      "mean_token_accuracy": 0.49295524060726165,
-      "step": 290
-    },
-    {
-      "epoch": 0.05010893246187364,
-      "mean_token_accuracy": 0.4976404458284378,
-      "step": 299,
+      "epoch": 0.09984871406959153,
+      "mean_token_accuracy": 0.24069985498984656,
+      "step": 33,
       "total_flos": 0.0,
-      "train_loss": 3.0202497909698995,
-      "train_runtime": 60.0371,
-      "train_samples_per_second": 79.507,
-      "train_steps_per_second": 4.98
+      "train_loss": 5.556107954545454,
+      "train_runtime": 94.197,
+      "train_samples_per_second": 11.222,
+      "train_steps_per_second": 0.35
     }
   ],
   "logging_steps": 10,
-  "max_steps": 299,
+  "max_steps": 33,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdd0e83665f82cbff34945a1dc5f700141602e90d62707ec1ecd8225c96d8e93
+oid sha256:868f332cb0455cb6b6311de1634eb90272c59e27c7f7a526d31c36d9ac33732f
 size 7352