End of training

README.md

@@ -6,6 +6,8 @@ model_name: gpt2-RMT-2
 tags:
 - generated_from_trainer
 - open-r1
+- trl
+- sft
 licence: license
 ---
 
@@ -27,7 +29,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/j0nfe37z) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/lhnwfjsu) 
 
 
 This model was trained with SFT.

all_results.json

@@ -1,12 +1,12 @@
 {
-    "eval_runtime": 0.2685,
+    "eval_runtime": 0.2372,
     "eval_samples": 100,
-    "eval_samples_per_second": 178.767,
-    "eval_steps_per_second": 11.173,
-    "total_flos": 0.0,
-    "train_loss": 5.5703125,
-    "train_runtime": 94.29,
+    "eval_samples_per_second": 202.381,
+    "eval_steps_per_second": 12.649,
+    "total_flos": 541940076511232.0,
+    "train_loss": 5.5710227272727275,
+    "train_runtime": 94.1692,
     "train_samples": 19883,
-    "train_samples_per_second": 11.211,
+    "train_samples_per_second": 11.226,
     "train_steps_per_second": 0.35
 }

config.json

@@ -1,22 +1,28 @@
 {
-  "_attn_implementation_autoset": true,
   "activation_function": "gelu_new",
+  "align": "left",
   "architectures": [
-    "GPT2LMHeadModel"
+    "RecurrentWrapper"
   ],
   "attn_pdrop": 0.1,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
+  "input_seg_len": 1004,
+  "is_memory_all": false,
   "layer_norm_epsilon": 1e-05,
-  "model_type": "gpt2",
+  "max_n_segments": 2,
+  "memory_size": 10,
+  "model_type": "rmt",
   "n_ctx": 1024,
   "n_embd": 768,
   "n_head": 12,
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "num_mem_tokens": 10,
+  "output_seg_len": 1004,
   "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
   "scale_attn_by_inverse_layer_idx": false,

eval_results.json

@@ -1,6 +1,6 @@
 {
-    "eval_runtime": 0.2685,
+    "eval_runtime": 0.2372,
     "eval_samples": 100,
-    "eval_samples_per_second": 178.767,
-    "eval_steps_per_second": 11.173
+    "eval_samples_per_second": 202.381,
+    "eval_steps_per_second": 12.649
 }

generation_config.json

@@ -2,5 +2,6 @@
   "_from_model_config": true,
   "bos_token_id": 50256,
   "eos_token_id": 50256,
-  "transformers_version": "4.50.0.dev0"
+  "transformers_version": "4.50.0.dev0",
+  "use_cache": false
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddafb83cf1a59157af80d355275bd09239a37d622ab84d9c4df9fd1bb357cbce
-size 326089656
+oid sha256:9f2c2cb3ce04aca9a075cf3e13f0f9653bf4b8a7da1160df52d27a2d3ee0ceb3
+size 248912768

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 0.0,
-    "train_loss": 5.5703125,
-    "train_runtime": 94.29,
+    "total_flos": 541940076511232.0,
+    "train_loss": 5.5710227272727275,
+    "train_runtime": 94.1692,
     "train_samples": 19883,
-    "train_samples_per_second": 11.211,
+    "train_samples_per_second": 11.226,
     "train_steps_per_second": 0.35
 }

trainer_state.json

@@ -10,36 +10,36 @@
   "log_history": [
     {
       "epoch": 0.030257186081694403,
-      "grad_norm": 23.19076749983313,
+      "grad_norm": 19.446511460556057,
       "learning_rate": 1.796093065705644e-05,
-      "loss": 6.8891,
-      "mean_token_accuracy": 0.14328600466251373,
+      "loss": 6.9188,
+      "mean_token_accuracy": 0.14323783591389655,
       "step": 10
     },
     {
       "epoch": 0.060514372163388806,
-      "grad_norm": 10.552981359945921,
+      "grad_norm": 10.827592460467098,
       "learning_rate": 8.382180034472353e-06,
-      "loss": 5.3461,
-      "mean_token_accuracy": 0.1898418039083481,
+      "loss": 5.3359,
+      "mean_token_accuracy": 0.19127428904175758,
       "step": 20
     },
     {
       "epoch": 0.0907715582450832,
-      "grad_norm": 8.191712743407091,
+      "grad_norm": 8.615023934478028,
       "learning_rate": 5.234682881719766e-07,
-      "loss": 4.75,
-      "mean_token_accuracy": 0.2335878863930702,
+      "loss": 4.7375,
+      "mean_token_accuracy": 0.2355961874127388,
       "step": 30
     },
     {
       "epoch": 0.09984871406959153,
-      "mean_token_accuracy": 0.23857259502013525,
+      "mean_token_accuracy": 0.2397766982515653,
       "step": 33,
-      "total_flos": 0.0,
-      "train_loss": 5.5703125,
-      "train_runtime": 94.29,
-      "train_samples_per_second": 11.211,
+      "total_flos": 541940076511232.0,
+      "train_loss": 5.5710227272727275,
+      "train_runtime": 94.1692,
+      "train_samples_per_second": 11.226,
       "train_steps_per_second": 0.35
     }
   ],
@@ -60,7 +60,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 0.0,
+  "total_flos": 541940076511232.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5fa711d6cdcb0b2b63e61e4b8e967fec2988c80fb803ba46477961ca838fb4d
+oid sha256:a1d79e0dd26daa466cbf9b51285fc1b10493e4f85980a882ccde209f5af06f5c
 size 7352