Model save

README.md

@@ -1,17 +1,17 @@
 ---
 base_model: openai-community/gpt2
-datasets: HuggingFaceFW/fineweb-edu
 library_name: transformers
 model_name: gpt2-RMT-2
 tags:
 - generated_from_trainer
-- open-r1
+- trl
+- sft
 licence: license
 ---
 
 # Model Card for gpt2-RMT-2
 
-This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceFW/fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/qz7kfrzp) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/qothcfg8) 
 
 
 This model was trained with SFT.

all_results.json

@@ -3,10 +3,10 @@
     "eval_samples": 100,
     "eval_samples_per_second": 99.696,
     "eval_steps_per_second": 12.462,
-    "total_flos": 0.0,
-    "train_loss": 4.772621268656716,
-    "train_runtime": 29.2884,
+    "total_flos": 550151290617856.0,
+    "train_loss": 4.790811567164179,
+    "train_runtime": 29.3533,
     "train_samples": 19883,
-    "train_samples_per_second": 36.093,
-    "train_steps_per_second": 2.288
+    "train_samples_per_second": 36.013,
+    "train_steps_per_second": 2.283
 }

config.json

@@ -1,8 +1,7 @@
 {
-  "_attn_implementation_autoset": true,
   "activation_function": "gelu_new",
   "architectures": [
-    "GPT2LMHeadModel"
+    "RecurrentWrapper"
   ],
   "attn_pdrop": 0.1,
   "bos_token_id": 50256,
@@ -32,7 +31,8 @@
       "max_length": 50
     }
   },
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.50.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 50257
 }

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.50.0.dev0"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e3c2740535e03f24f5cb574ba61bf881f6b6c33883fc13ee4948a731051a5ab
-size 326110968
+oid sha256:8eead2c9b2060803ef718a002144657eb03b943597178519430d4167776a9619
+size 248912768

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 0.0,
-    "train_loss": 4.772621268656716,
-    "train_runtime": 29.2884,
+    "total_flos": 550151290617856.0,
+    "train_loss": 4.790811567164179,
+    "train_runtime": 29.3533,
     "train_samples": 19883,
-    "train_samples_per_second": 36.093,
-    "train_steps_per_second": 2.288
+    "train_samples_per_second": 36.013,
+    "train_steps_per_second": 2.283
 }

trainer_state.json

@@ -10,61 +10,61 @@
   "log_history": [
     {
       "epoch": 0.015128593040847202,
-      "grad_norm": 33.7647987029562,
+      "grad_norm": 31.61330947025634,
       "learning_rate": 1.9876883405951378e-05,
-      "loss": 7.125,
-      "mean_token_accuracy": 0.14012572318315505,
+      "loss": 7.1484,
+      "mean_token_accuracy": 0.1398914076387882,
       "step": 10
     },
     {
       "epoch": 0.030257186081694403,
-      "grad_norm": 10.612758737576552,
+      "grad_norm": 10.03732149051733,
       "learning_rate": 1.777145961456971e-05,
-      "loss": 5.5062,
-      "mean_token_accuracy": 0.1818697050213814,
+      "loss": 5.5297,
+      "mean_token_accuracy": 0.18087630867958068,
       "step": 20
     },
     {
       "epoch": 0.0453857791225416,
-      "grad_norm": 5.133709750875046,
+      "grad_norm": 5.344272287258441,
       "learning_rate": 1.3583679495453e-05,
-      "loss": 4.6188,
-      "mean_token_accuracy": 0.24615719988942147,
+      "loss": 4.6375,
+      "mean_token_accuracy": 0.24480567798018454,
       "step": 30
     },
     {
       "epoch": 0.060514372163388806,
-      "grad_norm": 3.7566168018221653,
+      "grad_norm": 3.8172946073830114,
       "learning_rate": 8.43565534959769e-06,
-      "loss": 4.1492,
-      "mean_token_accuracy": 0.2883812889456749,
+      "loss": 4.1672,
+      "mean_token_accuracy": 0.28589935302734376,
       "step": 40
     },
     {
       "epoch": 0.07564296520423601,
-      "grad_norm": 3.0972835436341253,
+      "grad_norm": 3.077300488264507,
       "learning_rate": 3.7067960895016277e-06,
-      "loss": 3.9453,
-      "mean_token_accuracy": 0.30843553096055987,
+      "loss": 3.9633,
+      "mean_token_accuracy": 0.3073825791478157,
       "step": 50
     },
     {
       "epoch": 0.0907715582450832,
-      "grad_norm": 2.978044733724223,
+      "grad_norm": 2.9692193643977336,
       "learning_rate": 6.641957350279838e-07,
-      "loss": 3.8914,
-      "mean_token_accuracy": 0.3149686068296432,
+      "loss": 3.9047,
+      "mean_token_accuracy": 0.31336415857076644,
       "step": 60
     },
     {
       "epoch": 0.10136157337367625,
-      "mean_token_accuracy": 0.3118449641125543,
+      "mean_token_accuracy": 0.31038159344877514,
       "step": 67,
-      "total_flos": 0.0,
-      "train_loss": 4.772621268656716,
-      "train_runtime": 29.2884,
-      "train_samples_per_second": 36.093,
-      "train_steps_per_second": 2.288
+      "total_flos": 550151290617856.0,
+      "train_loss": 4.790811567164179,
+      "train_runtime": 29.3533,
+      "train_samples_per_second": 36.013,
+      "train_steps_per_second": 2.283
     }
   ],
   "logging_steps": 10,
@@ -84,7 +84,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 0.0,
+  "total_flos": 550151290617856.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be3f4e5fc95a8a097fc233c1e01227f2b9fb455b30c25f6543ddfec1b328f220
+oid sha256:68fc74c433f058a40ea1103a36e974aec030be05fef6c7bef96ba33c56290c81
 size 7352