End of training

README.md

@@ -29,7 +29,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/lhnwfjsu) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/s18574s18574-/huggingface/runs/5p39svg5) 
 
 
 This model was trained with SFT.

all_results.json

@@ -1,12 +1,12 @@
 {
-    "eval_runtime": 0.2372,
+    "eval_runtime": 0.2379,
     "eval_samples": 100,
-    "eval_samples_per_second": 202.381,
-    "eval_steps_per_second": 12.649,
+    "eval_samples_per_second": 201.753,
+    "eval_steps_per_second": 12.61,
     "total_flos": 541940076511232.0,
-    "train_loss": 5.5710227272727275,
-    "train_runtime": 94.1692,
+    "train_loss": 11.106060606060606,
+    "train_runtime": 94.1702,
     "train_samples": 19883,
-    "train_samples_per_second": 11.226,
+    "train_samples_per_second": 11.225,
     "train_steps_per_second": 0.35
 }

config.json

@@ -2,9 +2,103 @@
   "activation_function": "gelu_new",
   "align": "left",
   "architectures": [
-    "RecurrentWrapper"
+    "RecurrentMemoryTransformer"
   ],
   "attn_pdrop": 0.1,
+  "base_model_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "openai-community/gpt2",
+    "activation_function": "gelu_new",
+    "add_cross_attention": false,
+    "architectures": [
+      "GPT2LMHeadModel"
+    ],
+    "attn_pdrop": 0.1,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 50256,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "embd_pdrop": 0.1,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 50256,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_epsilon": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "gpt2",
+    "n_ctx": 1024,
+    "n_embd": 768,
+    "n_head": 12,
+    "n_inner": null,
+    "n_layer": 12,
+    "n_positions": 1024,
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "reorder_and_upcast_attn": false,
+    "repetition_penalty": 1.0,
+    "resid_pdrop": 0.1,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "scale_attn_by_inverse_layer_idx": false,
+    "scale_attn_weights": true,
+    "sep_token_id": null,
+    "summary_activation": null,
+    "summary_first_dropout": 0.1,
+    "summary_proj_to_labels": true,
+    "summary_type": "cls_index",
+    "summary_use_proj": true,
+    "suppress_tokens": null,
+    "task_specific_params": {
+      "text-generation": {
+        "do_sample": true,
+        "max_length": 50
+      }
+    },
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": false,
+    "vocab_size": 50257
+  },
+  "base_model_type": "gpt2",
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,

eval_results.json

@@ -1,6 +1,6 @@
 {
-    "eval_runtime": 0.2372,
+    "eval_runtime": 0.2379,
     "eval_samples": 100,
-    "eval_samples_per_second": 202.381,
-    "eval_steps_per_second": 12.649
+    "eval_samples_per_second": 201.753,
+    "eval_steps_per_second": 12.61
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f2c2cb3ce04aca9a075cf3e13f0f9653bf4b8a7da1160df52d27a2d3ee0ceb3
-size 248912768
+oid sha256:6a151f2bd4439693232966936e8838ed7a0f3181ee2be269d71c0f7dc5c91300
+size 248915448

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 541940076511232.0,
-    "train_loss": 5.5710227272727275,
-    "train_runtime": 94.1692,
+    "train_loss": 11.106060606060606,
+    "train_runtime": 94.1702,
     "train_samples": 19883,
-    "train_samples_per_second": 11.226,
+    "train_samples_per_second": 11.225,
     "train_steps_per_second": 0.35
 }

trainer_state.json

@@ -10,36 +10,36 @@
   "log_history": [
     {
       "epoch": 0.030257186081694403,
-      "grad_norm": 19.446511460556057,
+      "grad_norm": 35.22744368526962,
       "learning_rate": 1.796093065705644e-05,
-      "loss": 6.9188,
-      "mean_token_accuracy": 0.14323783591389655,
+      "loss": 13.7766,
+      "mean_token_accuracy": 0.1436693400144577,
       "step": 10
     },
     {
       "epoch": 0.060514372163388806,
-      "grad_norm": 10.827592460467098,
+      "grad_norm": 23.6320168043645,
       "learning_rate": 8.382180034472353e-06,
-      "loss": 5.3359,
-      "mean_token_accuracy": 0.19127428904175758,
+      "loss": 10.65,
+      "mean_token_accuracy": 0.19171493500471115,
       "step": 20
     },
     {
       "epoch": 0.0907715582450832,
-      "grad_norm": 8.615023934478028,
+      "grad_norm": 15.21905964248453,
       "learning_rate": 5.234682881719766e-07,
-      "loss": 4.7375,
-      "mean_token_accuracy": 0.2355961874127388,
+      "loss": 9.4422,
+      "mean_token_accuracy": 0.23619858622550965,
       "step": 30
     },
     {
       "epoch": 0.09984871406959153,
-      "mean_token_accuracy": 0.2397766982515653,
+      "mean_token_accuracy": 0.23997420817613602,
       "step": 33,
       "total_flos": 541940076511232.0,
-      "train_loss": 5.5710227272727275,
-      "train_runtime": 94.1692,
-      "train_samples_per_second": 11.226,
+      "train_loss": 11.106060606060606,
+      "train_runtime": 94.1702,
+      "train_samples_per_second": 11.225,
       "train_steps_per_second": 0.35
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1d79e0dd26daa466cbf9b51285fc1b10493e4f85980a882ccde209f5af06f5c
+oid sha256:edbeaeb7aa6557be5c809a22a09614c50689d24c266e9e86dfcd26bb1fbec880
 size 7352