Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: openai-community/gpt2
-datasets: HuggingFaceFW/fineweb-edu
 library_name: transformers
 model_name: gpt2-RMT-2-mem512
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - sft
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for gpt2-RMT-2-mem512
 
-This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on the [HuggingFaceFW/fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/shin2021001-osaka-city-university/huggingface/runs/gt0ble5n) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/shin2021001-osaka-city-university/huggingface/runs/nt4l8say) 
 
 
 This model was trained with SFT.

all_results.json

@@ -3,10 +3,10 @@
     "eval_samples": 100,
     "eval_samples_per_second": 249.241,
     "eval_steps_per_second": 16.08,
-    "total_flos": 5418484972388352.0,
-    "train_loss": 3.603820517774343,
-    "train_runtime": 400.5994,
+    "total_flos": 5419008396361728.0,
+    "train_loss": 4.076150745858688,
+    "train_runtime": 7573.4415,
     "train_samples": 19883,
-    "train_samples_per_second": 51.708,
-    "train_steps_per_second": 1.615
+    "train_samples_per_second": 87.49,
+    "train_steps_per_second": 2.734
 }

config.json

@@ -103,7 +103,7 @@
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "input_seg_len": 512,
+  "input_seg_len": 16,
   "is_memory_all": false,
   "layer_norm_epsilon": 1e-05,
   "max_n_segments": 2,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93df75f5f8b5b0835ad0330c6b3b9585fae38e86fa81ca6401d49acc12d7f317
+oid sha256:65cd08da7dc4048511a86bef339939ec8531258568d7775e32f42921c96aaab4
 size 248915448

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 5418484972388352.0,
-    "train_loss": 3.603820517774343,
-    "train_runtime": 400.5994,
+    "total_flos": 5419008396361728.0,
+    "train_loss": 4.076150745858688,
+    "train_runtime": 7573.4415,
     "train_samples": 19883,
-    "train_samples_per_second": 51.708,
-    "train_steps_per_second": 1.615
+    "train_samples_per_second": 87.49,
+    "train_steps_per_second": 2.734
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88846274ec0b368f3fe3ca616827474715cdc7f57eaee25769d05eff2d5e13d6
+oid sha256:3e506da221a187e12d0f07664922d412eb372d52d46c7a3b6e4d2d2ee1a0abcd
 size 7352