Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: openai-community/gpt2
-datasets: HuggingFaceFW/fineweb-edu
 library_name: transformers
 model_name: gpt2-RMT-2
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - sft
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for gpt2-RMT-2
 
-This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceFW/fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/qothcfg8) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/1xooe0sm) 
 
 
 This model was trained with SFT.

all_results.json

@@ -3,10 +3,10 @@
     "eval_samples": 100,
     "eval_samples_per_second": 100.273,
     "eval_steps_per_second": 12.534,
-    "total_flos": 550151290617856.0,
-    "train_loss": 4.790811567164179,
-    "train_runtime": 29.3533,
+    "total_flos": 541940076511232.0,
+    "train_loss": 5.546875,
+    "train_runtime": 36.3584,
     "train_samples": 19883,
-    "train_samples_per_second": 36.013,
-    "train_steps_per_second": 2.283
+    "train_samples_per_second": 29.074,
+    "train_steps_per_second": 0.908
 }

config.json

@@ -33,6 +33,6 @@
   },
   "torch_dtype": "bfloat16",
   "transformers_version": "4.50.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 50257
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8eead2c9b2060803ef718a002144657eb03b943597178519430d4167776a9619
+oid sha256:9ef6b24bad9283be8a3270f3898cb6e1d27404fa045313443fde712995a1cabc
 size 248912768

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 550151290617856.0,
-    "train_loss": 4.790811567164179,
-    "train_runtime": 29.3533,
+    "total_flos": 541940076511232.0,
+    "train_loss": 5.546875,
+    "train_runtime": 36.3584,
     "train_samples": 19883,
-    "train_samples_per_second": 36.013,
-    "train_steps_per_second": 2.283
+    "train_samples_per_second": 29.074,
+    "train_steps_per_second": 0.908
 }

trainer_state.json

@@ -1,74 +1,50 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.10136157337367625,
+  "epoch": 0.09984871406959153,
   "eval_steps": 100,
-  "global_step": 67,
+  "global_step": 33,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.015128593040847202,
-      "grad_norm": 31.61330947025634,
-      "learning_rate": 1.9876883405951378e-05,
-      "loss": 7.1484,
-      "mean_token_accuracy": 0.1398914076387882,
-      "step": 10
-    },
     {
       "epoch": 0.030257186081694403,
-      "grad_norm": 10.03732149051733,
-      "learning_rate": 1.777145961456971e-05,
-      "loss": 5.5297,
-      "mean_token_accuracy": 0.18087630867958068,
-      "step": 20
-    },
-    {
-      "epoch": 0.0453857791225416,
-      "grad_norm": 5.344272287258441,
-      "learning_rate": 1.3583679495453e-05,
-      "loss": 4.6375,
-      "mean_token_accuracy": 0.24480567798018454,
-      "step": 30
+      "grad_norm": 18.3522235886847,
+      "learning_rate": 1.796093065705644e-05,
+      "loss": 6.8937,
+      "mean_token_accuracy": 0.1434693343937397,
+      "step": 10
     },
     {
       "epoch": 0.060514372163388806,
-      "grad_norm": 3.8172946073830114,
-      "learning_rate": 8.43565534959769e-06,
-      "loss": 4.1672,
-      "mean_token_accuracy": 0.28589935302734376,
-      "step": 40
-    },
-    {
-      "epoch": 0.07564296520423601,
-      "grad_norm": 3.077300488264507,
-      "learning_rate": 3.7067960895016277e-06,
-      "loss": 3.9633,
-      "mean_token_accuracy": 0.3073825791478157,
-      "step": 50
+      "grad_norm": 10.614870433655174,
+      "learning_rate": 8.382180034472353e-06,
+      "loss": 5.3195,
+      "mean_token_accuracy": 0.1919048972427845,
+      "step": 20
     },
     {
       "epoch": 0.0907715582450832,
-      "grad_norm": 2.9692193643977336,
-      "learning_rate": 6.641957350279838e-07,
-      "loss": 3.9047,
-      "mean_token_accuracy": 0.31336415857076644,
-      "step": 60
+      "grad_norm": 7.571339027683935,
+      "learning_rate": 5.234682881719766e-07,
+      "loss": 4.707,
+      "mean_token_accuracy": 0.23758598268032075,
+      "step": 30
     },
     {
-      "epoch": 0.10136157337367625,
-      "mean_token_accuracy": 0.31038159344877514,
-      "step": 67,
-      "total_flos": 550151290617856.0,
-      "train_loss": 4.790811567164179,
-      "train_runtime": 29.3533,
-      "train_samples_per_second": 36.013,
-      "train_steps_per_second": 2.283
+      "epoch": 0.09984871406959153,
+      "mean_token_accuracy": 0.24127203722794852,
+      "step": 33,
+      "total_flos": 541940076511232.0,
+      "train_loss": 5.546875,
+      "train_runtime": 36.3584,
+      "train_samples_per_second": 29.074,
+      "train_steps_per_second": 0.908
     }
   ],
   "logging_steps": 10,
-  "max_steps": 67,
+  "max_steps": 33,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -84,7 +60,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 550151290617856.0,
+  "total_flos": 541940076511232.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68fc74c433f058a40ea1103a36e974aec030be05fef6c7bef96ba33c56290c81
+oid sha256:baf06d605ca2ac794fb0cb09d28d4e25826d2c1b68fdaaefca458cb0a917d847
 size 7352