Upload folder using huggingface_hub

checkpoint-1000/config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "EleutherAI/pythia-14m",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "partial_rotary_factor": 0.25,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

checkpoint-1000/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.45.2",
+  "use_cache": false
+}

checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:638d03da1db382b44f2350d86858b78e81ed954c6e0c6044d5be9c2e6a37b4ea
+size 56279344

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef85159f1d30dfa3094212aebace3966606a5a9aa0cc5abb4ae66ba5a37de000
+size 112605707

checkpoint-1000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff83006d85c789a73e3d5962b60a1ea20ea38374ebe54f6d5e8a5c5777efbb05
+size 16325

checkpoint-1000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4405d2b9aa7065e98bf1b965702b69709bef30d31b3257e2445623659424edc
+size 16325

checkpoint-1000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d63c51a9642e331eb3ab07696e2e11c88c29f4f34cf4f99f8ffa4fb55af453b
+size 16325

checkpoint-1000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba4e5ea853aa4898edd412a4da34a96b758f0f7218655949614277cb1910eb31
+size 16325

checkpoint-1000/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481092ba472bbb5b59655c5d961d7f139eda9daf3bb24d9d6f7ef405894f2153
+size 16325

checkpoint-1000/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93e9df5970151da7ccada8e10c311cc4a6237c90a1701b6a5187bce10af76bd8
+size 16325

checkpoint-1000/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56e3382b0cee8a47b032dacb4635c6434a00b08b20843e7882b8fafa689b2bf0
+size 16325

checkpoint-1000/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e71863697b5311ef5c1666e4a06796a8f027fab8efa2dfff79a823e493506448
+size 16325

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5b05955bb197697f5a716e4c344035e12a4a81599fd3ab5b82077b37e185f6c
+size 1465

checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,214 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-1000/trainer_state.json

@@ -0,0 +1,110 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.03999840006399744,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 3.999840006399744e-05,
+      "grad_norm": 1.4777933359146118,
+      "learning_rate": 0.001,
+      "loss": 10.8399,
+      "step": 1
+    },
+    {
+      "epoch": 0.003999840006399744,
+      "grad_norm": 0.22688856720924377,
+      "learning_rate": 0.001,
+      "loss": 7.558,
+      "step": 100
+    },
+    {
+      "epoch": 0.007999680012799487,
+      "grad_norm": 0.24372541904449463,
+      "learning_rate": 0.001,
+      "loss": 6.3052,
+      "step": 200
+    },
+    {
+      "epoch": 0.011999520019199232,
+      "grad_norm": 0.5153403282165527,
+      "learning_rate": 0.001,
+      "loss": 5.9045,
+      "step": 300
+    },
+    {
+      "epoch": 0.015999360025598975,
+      "grad_norm": 0.5177266597747803,
+      "learning_rate": 0.001,
+      "loss": 5.6063,
+      "step": 400
+    },
+    {
+      "epoch": 0.01999920003199872,
+      "grad_norm": 0.6018334627151489,
+      "learning_rate": 0.001,
+      "loss": 5.4024,
+      "step": 500
+    },
+    {
+      "epoch": 0.023999040038398464,
+      "grad_norm": 0.4220522940158844,
+      "learning_rate": 0.001,
+      "loss": 5.2604,
+      "step": 600
+    },
+    {
+      "epoch": 0.027998880044798207,
+      "grad_norm": 0.4826813042163849,
+      "learning_rate": 0.001,
+      "loss": 5.149,
+      "step": 700
+    },
+    {
+      "epoch": 0.03199872005119795,
+      "grad_norm": 0.5880510807037354,
+      "learning_rate": 0.001,
+      "loss": 5.0663,
+      "step": 800
+    },
+    {
+      "epoch": 0.035998560057597696,
+      "grad_norm": 0.4875524640083313,
+      "learning_rate": 0.001,
+      "loss": 5.0008,
+      "step": 900
+    },
+    {
+      "epoch": 0.03999840006399744,
+      "grad_norm": 0.526023805141449,
+      "learning_rate": 0.001,
+      "loss": 4.9403,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 150000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.39981297664e+16,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b573a96e19aa9061cdc1bcf783263829ca1386cefdf168bd82b7d605ad2594ec
+size 5649

checkpoint-10000/config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "EleutherAI/pythia-14m",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "partial_rotary_factor": 0.25,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

checkpoint-10000/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.45.2",
+  "use_cache": false
+}

checkpoint-10000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1ba695e4e6f44c1c0039e6216a4d1b76d0b985b30ea80db415612ae547d6a44
+size 56279344

checkpoint-10000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b1444ba3096ae4e5476e2001d41a571ada2256016444f62e6954c0ef46f5798
+size 112605707

checkpoint-10000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff83006d85c789a73e3d5962b60a1ea20ea38374ebe54f6d5e8a5c5777efbb05
+size 16325

checkpoint-10000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4405d2b9aa7065e98bf1b965702b69709bef30d31b3257e2445623659424edc
+size 16325

checkpoint-10000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d63c51a9642e331eb3ab07696e2e11c88c29f4f34cf4f99f8ffa4fb55af453b
+size 16325

checkpoint-10000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba4e5ea853aa4898edd412a4da34a96b758f0f7218655949614277cb1910eb31
+size 16325

checkpoint-10000/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481092ba472bbb5b59655c5d961d7f139eda9daf3bb24d9d6f7ef405894f2153
+size 16325

checkpoint-10000/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93e9df5970151da7ccada8e10c311cc4a6237c90a1701b6a5187bce10af76bd8
+size 16325

checkpoint-10000/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56e3382b0cee8a47b032dacb4635c6434a00b08b20843e7882b8fafa689b2bf0
+size 16325

checkpoint-10000/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e71863697b5311ef5c1666e4a06796a8f027fab8efa2dfff79a823e493506448
+size 16325

checkpoint-10000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a70183c36c2f7607bce16c3b85fa32b96b09ecfb511ff7236bc48e687d6b1c7
+size 1465

checkpoint-10000/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-10000/tokenizer_config.json

@@ -0,0 +1,214 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-10000/trainer_state.json

@@ -0,0 +1,740 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.3999840006399744,
+  "eval_steps": 500,
+  "global_step": 10000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 3.999840006399744e-05,
+      "grad_norm": 1.4777933359146118,
+      "learning_rate": 0.001,
+      "loss": 10.8399,
+      "step": 1
+    },
+    {
+      "epoch": 0.003999840006399744,
+      "grad_norm": 0.22688856720924377,
+      "learning_rate": 0.001,
+      "loss": 7.558,
+      "step": 100
+    },
+    {
+      "epoch": 0.007999680012799487,
+      "grad_norm": 0.24372541904449463,
+      "learning_rate": 0.001,
+      "loss": 6.3052,
+      "step": 200
+    },
+    {
+      "epoch": 0.011999520019199232,
+      "grad_norm": 0.5153403282165527,
+      "learning_rate": 0.001,
+      "loss": 5.9045,
+      "step": 300
+    },
+    {
+      "epoch": 0.015999360025598975,
+      "grad_norm": 0.5177266597747803,
+      "learning_rate": 0.001,
+      "loss": 5.6063,
+      "step": 400
+    },
+    {
+      "epoch": 0.01999920003199872,
+      "grad_norm": 0.6018334627151489,
+      "learning_rate": 0.001,
+      "loss": 5.4024,
+      "step": 500
+    },
+    {
+      "epoch": 0.023999040038398464,
+      "grad_norm": 0.4220522940158844,
+      "learning_rate": 0.001,
+      "loss": 5.2604,
+      "step": 600
+    },
+    {
+      "epoch": 0.027998880044798207,
+      "grad_norm": 0.4826813042163849,
+      "learning_rate": 0.001,
+      "loss": 5.149,
+      "step": 700
+    },
+    {
+      "epoch": 0.03199872005119795,
+      "grad_norm": 0.5880510807037354,
+      "learning_rate": 0.001,
+      "loss": 5.0663,
+      "step": 800
+    },
+    {
+      "epoch": 0.035998560057597696,
+      "grad_norm": 0.4875524640083313,
+      "learning_rate": 0.001,
+      "loss": 5.0008,
+      "step": 900
+    },
+    {
+      "epoch": 0.03999840006399744,
+      "grad_norm": 0.526023805141449,
+      "learning_rate": 0.001,
+      "loss": 4.9403,
+      "step": 1000
+    },
+    {
+      "epoch": 0.04399824007039718,
+      "grad_norm": 0.4925878942012787,
+      "learning_rate": 0.001,
+      "loss": 4.8988,
+      "step": 1100
+    },
+    {
+      "epoch": 0.04799808007679693,
+      "grad_norm": 0.5051125884056091,
+      "learning_rate": 0.001,
+      "loss": 4.8552,
+      "step": 1200
+    },
+    {
+      "epoch": 0.051997920083196675,
+      "grad_norm": 0.6583888530731201,
+      "learning_rate": 0.001,
+      "loss": 4.8247,
+      "step": 1300
+    },
+    {
+      "epoch": 0.055997760089596414,
+      "grad_norm": 0.4777211546897888,
+      "learning_rate": 0.001,
+      "loss": 4.7899,
+      "step": 1400
+    },
+    {
+      "epoch": 0.05999760009599616,
+      "grad_norm": 0.5130186080932617,
+      "learning_rate": 0.001,
+      "loss": 4.7612,
+      "step": 1500
+    },
+    {
+      "epoch": 0.0639974401023959,
+      "grad_norm": 0.5315548777580261,
+      "learning_rate": 0.001,
+      "loss": 4.7361,
+      "step": 1600
+    },
+    {
+      "epoch": 0.06799728010879565,
+      "grad_norm": 0.47726863622665405,
+      "learning_rate": 0.001,
+      "loss": 4.7013,
+      "step": 1700
+    },
+    {
+      "epoch": 0.07199712011519539,
+      "grad_norm": 0.5642004013061523,
+      "learning_rate": 0.001,
+      "loss": 4.6645,
+      "step": 1800
+    },
+    {
+      "epoch": 0.07599696012159514,
+      "grad_norm": 0.545949399471283,
+      "learning_rate": 0.001,
+      "loss": 4.6219,
+      "step": 1900
+    },
+    {
+      "epoch": 0.07999680012799489,
+      "grad_norm": 0.5206509232521057,
+      "learning_rate": 0.001,
+      "loss": 4.5969,
+      "step": 2000
+    },
+    {
+      "epoch": 0.08399664013439462,
+      "grad_norm": 0.44842126965522766,
+      "learning_rate": 0.001,
+      "loss": 4.5708,
+      "step": 2100
+    },
+    {
+      "epoch": 0.08799648014079436,
+      "grad_norm": 0.6365007162094116,
+      "learning_rate": 0.001,
+      "loss": 4.5523,
+      "step": 2200
+    },
+    {
+      "epoch": 0.09199632014719411,
+      "grad_norm": 0.5729643702507019,
+      "learning_rate": 0.001,
+      "loss": 4.5305,
+      "step": 2300
+    },
+    {
+      "epoch": 0.09599616015359386,
+      "grad_norm": 0.5888395309448242,
+      "learning_rate": 0.001,
+      "loss": 4.5189,
+      "step": 2400
+    },
+    {
+      "epoch": 0.0999960001599936,
+      "grad_norm": 0.6099081039428711,
+      "learning_rate": 0.001,
+      "loss": 4.5038,
+      "step": 2500
+    },
+    {
+      "epoch": 0.10399584016639335,
+      "grad_norm": 0.5354722738265991,
+      "learning_rate": 0.001,
+      "loss": 4.487,
+      "step": 2600
+    },
+    {
+      "epoch": 0.10799568017279308,
+      "grad_norm": 0.6929482221603394,
+      "learning_rate": 0.001,
+      "loss": 4.4784,
+      "step": 2700
+    },
+    {
+      "epoch": 0.11199552017919283,
+      "grad_norm": 0.5511060357093811,
+      "learning_rate": 0.001,
+      "loss": 4.4682,
+      "step": 2800
+    },
+    {
+      "epoch": 0.11599536018559257,
+      "grad_norm": 0.5773873329162598,
+      "learning_rate": 0.001,
+      "loss": 4.4589,
+      "step": 2900
+    },
+    {
+      "epoch": 0.11999520019199232,
+      "grad_norm": 0.6061297655105591,
+      "learning_rate": 0.001,
+      "loss": 4.452,
+      "step": 3000
+    },
+    {
+      "epoch": 0.12399504019839207,
+      "grad_norm": 0.6082037687301636,
+      "learning_rate": 0.001,
+      "loss": 4.443,
+      "step": 3100
+    },
+    {
+      "epoch": 0.1279948802047918,
+      "grad_norm": 0.5562213063240051,
+      "learning_rate": 0.001,
+      "loss": 4.4342,
+      "step": 3200
+    },
+    {
+      "epoch": 0.13199472021119155,
+      "grad_norm": 0.5706282258033752,
+      "learning_rate": 0.001,
+      "loss": 4.4281,
+      "step": 3300
+    },
+    {
+      "epoch": 0.1359945602175913,
+      "grad_norm": 0.6546366810798645,
+      "learning_rate": 0.001,
+      "loss": 4.4192,
+      "step": 3400
+    },
+    {
+      "epoch": 0.13999440022399104,
+      "grad_norm": 0.5441614389419556,
+      "learning_rate": 0.001,
+      "loss": 4.4157,
+      "step": 3500
+    },
+    {
+      "epoch": 0.14399424023039079,
+      "grad_norm": 0.547122061252594,
+      "learning_rate": 0.001,
+      "loss": 4.4079,
+      "step": 3600
+    },
+    {
+      "epoch": 0.14799408023679053,
+      "grad_norm": 0.520165741443634,
+      "learning_rate": 0.001,
+      "loss": 4.4047,
+      "step": 3700
+    },
+    {
+      "epoch": 0.15199392024319028,
+      "grad_norm": 0.529140293598175,
+      "learning_rate": 0.001,
+      "loss": 4.3997,
+      "step": 3800
+    },
+    {
+      "epoch": 0.15599376024959002,
+      "grad_norm": 0.5902653336524963,
+      "learning_rate": 0.001,
+      "loss": 4.3921,
+      "step": 3900
+    },
+    {
+      "epoch": 0.15999360025598977,
+      "grad_norm": 0.6136724948883057,
+      "learning_rate": 0.001,
+      "loss": 4.3877,
+      "step": 4000
+    },
+    {
+      "epoch": 0.16399344026238952,
+      "grad_norm": 0.5538173913955688,
+      "learning_rate": 0.001,
+      "loss": 4.383,
+      "step": 4100
+    },
+    {
+      "epoch": 0.16799328026878924,
+      "grad_norm": 0.5516422986984253,
+      "learning_rate": 0.001,
+      "loss": 4.3815,
+      "step": 4200
+    },
+    {
+      "epoch": 0.17199312027518898,
+      "grad_norm": 0.535236656665802,
+      "learning_rate": 0.001,
+      "loss": 4.375,
+      "step": 4300
+    },
+    {
+      "epoch": 0.17599296028158873,
+      "grad_norm": 0.5214977860450745,
+      "learning_rate": 0.001,
+      "loss": 4.3716,
+      "step": 4400
+    },
+    {
+      "epoch": 0.17999280028798847,
+      "grad_norm": 0.473036527633667,
+      "learning_rate": 0.001,
+      "loss": 4.3679,
+      "step": 4500
+    },
+    {
+      "epoch": 0.18399264029438822,
+      "grad_norm": 0.5115819573402405,
+      "learning_rate": 0.001,
+      "loss": 4.3661,
+      "step": 4600
+    },
+    {
+      "epoch": 0.18799248030078797,
+      "grad_norm": 0.5074037313461304,
+      "learning_rate": 0.001,
+      "loss": 4.3606,
+      "step": 4700
+    },
+    {
+      "epoch": 0.1919923203071877,
+      "grad_norm": 0.4992203116416931,
+      "learning_rate": 0.001,
+      "loss": 4.3579,
+      "step": 4800
+    },
+    {
+      "epoch": 0.19599216031358746,
+      "grad_norm": 0.5755491256713867,
+      "learning_rate": 0.001,
+      "loss": 4.3501,
+      "step": 4900
+    },
+    {
+      "epoch": 0.1999920003199872,
+      "grad_norm": 0.5061823725700378,
+      "learning_rate": 0.001,
+      "loss": 4.3486,
+      "step": 5000
+    },
+    {
+      "epoch": 0.20399184032638695,
+      "grad_norm": 0.5323928594589233,
+      "learning_rate": 0.001,
+      "loss": 4.3477,
+      "step": 5100
+    },
+    {
+      "epoch": 0.2079916803327867,
+      "grad_norm": 0.5301046967506409,
+      "learning_rate": 0.001,
+      "loss": 4.3411,
+      "step": 5200
+    },
+    {
+      "epoch": 0.21199152033918645,
+      "grad_norm": 0.563123345375061,
+      "learning_rate": 0.001,
+      "loss": 4.3405,
+      "step": 5300
+    },
+    {
+      "epoch": 0.21599136034558616,
+      "grad_norm": 0.4736695885658264,
+      "learning_rate": 0.001,
+      "loss": 4.3391,
+      "step": 5400
+    },
+    {
+      "epoch": 0.2199912003519859,
+      "grad_norm": 0.46568775177001953,
+      "learning_rate": 0.001,
+      "loss": 4.3372,
+      "step": 5500
+    },
+    {
+      "epoch": 0.22399104035838566,
+      "grad_norm": 0.591935396194458,
+      "learning_rate": 0.001,
+      "loss": 4.3318,
+      "step": 5600
+    },
+    {
+      "epoch": 0.2279908803647854,
+      "grad_norm": 0.4887066185474396,
+      "learning_rate": 0.001,
+      "loss": 4.3318,
+      "step": 5700
+    },
+    {
+      "epoch": 0.23199072037118515,
+      "grad_norm": 0.4820877015590668,
+      "learning_rate": 0.001,
+      "loss": 4.3253,
+      "step": 5800
+    },
+    {
+      "epoch": 0.2359905603775849,
+      "grad_norm": 0.48746803402900696,
+      "learning_rate": 0.001,
+      "loss": 4.3242,
+      "step": 5900
+    },
+    {
+      "epoch": 0.23999040038398464,
+      "grad_norm": 0.5339901447296143,
+      "learning_rate": 0.001,
+      "loss": 4.3226,
+      "step": 6000
+    },
+    {
+      "epoch": 0.2439902403903844,
+      "grad_norm": 0.5406463146209717,
+      "learning_rate": 0.001,
+      "loss": 4.3197,
+      "step": 6100
+    },
+    {
+      "epoch": 0.24799008039678413,
+      "grad_norm": 0.5854783654212952,
+      "learning_rate": 0.001,
+      "loss": 4.318,
+      "step": 6200
+    },
+    {
+      "epoch": 0.25198992040318385,
+      "grad_norm": 0.47678840160369873,
+      "learning_rate": 0.001,
+      "loss": 4.3187,
+      "step": 6300
+    },
+    {
+      "epoch": 0.2559897604095836,
+      "grad_norm": 0.5523233413696289,
+      "learning_rate": 0.001,
+      "loss": 4.3117,
+      "step": 6400
+    },
+    {
+      "epoch": 0.25998960041598335,
+      "grad_norm": 0.46079719066619873,
+      "learning_rate": 0.001,
+      "loss": 4.3165,
+      "step": 6500
+    },
+    {
+      "epoch": 0.2639894404223831,
+      "grad_norm": 0.4956238269805908,
+      "learning_rate": 0.001,
+      "loss": 4.3116,
+      "step": 6600
+    },
+    {
+      "epoch": 0.26798928042878284,
+      "grad_norm": 0.532508373260498,
+      "learning_rate": 0.001,
+      "loss": 4.307,
+      "step": 6700
+    },
+    {
+      "epoch": 0.2719891204351826,
+      "grad_norm": 0.5244960188865662,
+      "learning_rate": 0.001,
+      "loss": 4.309,
+      "step": 6800
+    },
+    {
+      "epoch": 0.27598896044158233,
+      "grad_norm": 0.48627936840057373,
+      "learning_rate": 0.001,
+      "loss": 4.3025,
+      "step": 6900
+    },
+    {
+      "epoch": 0.2799888004479821,
+      "grad_norm": 0.5393197536468506,
+      "learning_rate": 0.001,
+      "loss": 4.3062,
+      "step": 7000
+    },
+    {
+      "epoch": 0.2839886404543818,
+      "grad_norm": 0.5384635925292969,
+      "learning_rate": 0.001,
+      "loss": 4.2993,
+      "step": 7100
+    },
+    {
+      "epoch": 0.28798848046078157,
+      "grad_norm": 0.5455852150917053,
+      "learning_rate": 0.001,
+      "loss": 4.299,
+      "step": 7200
+    },
+    {
+      "epoch": 0.2919883204671813,
+      "grad_norm": 0.45626768469810486,
+      "learning_rate": 0.001,
+      "loss": 4.2983,
+      "step": 7300
+    },
+    {
+      "epoch": 0.29598816047358106,
+      "grad_norm": 0.5111705660820007,
+      "learning_rate": 0.001,
+      "loss": 4.2992,
+      "step": 7400
+    },
+    {
+      "epoch": 0.2999880004799808,
+      "grad_norm": 0.5607153177261353,
+      "learning_rate": 0.001,
+      "loss": 4.291,
+      "step": 7500
+    },
+    {
+      "epoch": 0.30398784048638056,
+      "grad_norm": 0.5863308906555176,
+      "learning_rate": 0.001,
+      "loss": 4.2928,
+      "step": 7600
+    },
+    {
+      "epoch": 0.3079876804927803,
+      "grad_norm": 0.48837366700172424,
+      "learning_rate": 0.001,
+      "loss": 4.2951,
+      "step": 7700
+    },
+    {
+      "epoch": 0.31198752049918005,
+      "grad_norm": 0.5300312042236328,
+      "learning_rate": 0.001,
+      "loss": 4.2891,
+      "step": 7800
+    },
+    {
+      "epoch": 0.3159873605055798,
+      "grad_norm": 0.4869995415210724,
+      "learning_rate": 0.001,
+      "loss": 4.2868,
+      "step": 7900
+    },
+    {
+      "epoch": 0.31998720051197954,
+      "grad_norm": 0.5378390550613403,
+      "learning_rate": 0.001,
+      "loss": 4.2863,
+      "step": 8000
+    },
+    {
+      "epoch": 0.3239870405183793,
+      "grad_norm": 0.49154022336006165,
+      "learning_rate": 0.001,
+      "loss": 4.2902,
+      "step": 8100
+    },
+    {
+      "epoch": 0.32798688052477903,
+      "grad_norm": 0.5984882712364197,
+      "learning_rate": 0.001,
+      "loss": 4.2851,
+      "step": 8200
+    },
+    {
+      "epoch": 0.3319867205311788,
+      "grad_norm": 0.5132819414138794,
+      "learning_rate": 0.001,
+      "loss": 4.2854,
+      "step": 8300
+    },
+    {
+      "epoch": 0.33598656053757847,
+      "grad_norm": 0.5187487006187439,
+      "learning_rate": 0.001,
+      "loss": 4.2853,
+      "step": 8400
+    },
+    {
+      "epoch": 0.3399864005439782,
+      "grad_norm": 0.5310469269752502,
+      "learning_rate": 0.001,
+      "loss": 4.2785,
+      "step": 8500
+    },
+    {
+      "epoch": 0.34398624055037796,
+      "grad_norm": 0.5427576899528503,
+      "learning_rate": 0.001,
+      "loss": 4.2799,
+      "step": 8600
+    },
+    {
+      "epoch": 0.3479860805567777,
+      "grad_norm": 0.566967248916626,
+      "learning_rate": 0.001,
+      "loss": 4.2822,
+      "step": 8700
+    },
+    {
+      "epoch": 0.35198592056317746,
+      "grad_norm": 0.48460662364959717,
+      "learning_rate": 0.001,
+      "loss": 4.2772,
+      "step": 8800
+    },
+    {
+      "epoch": 0.3559857605695772,
+      "grad_norm": 0.5119643807411194,
+      "learning_rate": 0.001,
+      "loss": 4.2782,
+      "step": 8900
+    },
+    {
+      "epoch": 0.35998560057597695,
+      "grad_norm": 0.5386670827865601,
+      "learning_rate": 0.001,
+      "loss": 4.2749,
+      "step": 9000
+    },
+    {
+      "epoch": 0.3639854405823767,
+      "grad_norm": 0.5337742567062378,
+      "learning_rate": 0.001,
+      "loss": 4.2745,
+      "step": 9100
+    },
+    {
+      "epoch": 0.36798528058877644,
+      "grad_norm": 0.522514283657074,
+      "learning_rate": 0.001,
+      "loss": 4.2717,
+      "step": 9200
+    },
+    {
+      "epoch": 0.3719851205951762,
+      "grad_norm": 0.549670934677124,
+      "learning_rate": 0.001,
+      "loss": 4.272,
+      "step": 9300
+    },
+    {
+      "epoch": 0.37598496060157593,
+      "grad_norm": 0.6063629388809204,
+      "learning_rate": 0.001,
+      "loss": 4.2684,
+      "step": 9400
+    },
+    {
+      "epoch": 0.3799848006079757,
+      "grad_norm": 0.5420516133308411,
+      "learning_rate": 0.001,
+      "loss": 4.2674,
+      "step": 9500
+    },
+    {
+      "epoch": 0.3839846406143754,
+      "grad_norm": 0.48435646295547485,
+      "learning_rate": 0.001,
+      "loss": 4.2675,
+      "step": 9600
+    },
+    {
+      "epoch": 0.3879844806207752,
+      "grad_norm": 0.5914377570152283,
+      "learning_rate": 0.001,
+      "loss": 4.2737,
+      "step": 9700
+    },
+    {
+      "epoch": 0.3919843206271749,
+      "grad_norm": 0.4351874589920044,
+      "learning_rate": 0.001,
+      "loss": 4.2684,
+      "step": 9800
+    },
+    {
+      "epoch": 0.39598416063357467,
+      "grad_norm": 0.5514108538627625,
+      "learning_rate": 0.001,
+      "loss": 4.2671,
+      "step": 9900
+    },
+    {
+      "epoch": 0.3999840006399744,
+      "grad_norm": 0.5687771439552307,
+      "learning_rate": 0.001,
+      "loss": 4.2626,
+      "step": 10000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 150000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.39981297664e+17,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-10000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b573a96e19aa9061cdc1bcf783263829ca1386cefdf168bd82b7d605ad2594ec
+size 5649

checkpoint-100000/config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "EleutherAI/pythia-14m",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "partial_rotary_factor": 0.25,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

checkpoint-100000/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.45.2",
+  "use_cache": false
+}

checkpoint-100000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:474b712d999a8985525239ef03051e255271c60e094376cf1f927ae8a0af91b3
+size 56279344

checkpoint-100000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59886b2ae140cbe8da64a2239a400d34c37c1431bd61ad52842539d86f688120
+size 112605707

checkpoint-100000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daf26c930fc778c4127c596aa27bd243fe34e1aafd70f1c2e2344fd17a0fc5ae
+size 16325

checkpoint-100000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:677c13f0cc0bb08bbb080fdfeb71469f745f9a89f482907cfcfbbd31f924ed81
+size 16325

checkpoint-100000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:111bd2f8a56e8291b195da2889303ad84ea2144dc8b5d9bd9a2554aee0c6ed4d
+size 16325

checkpoint-100000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d4b38e6d4e20fbc05cee147d55111e549ff1b673bf6664138e2dce7b2baabf
+size 16325

checkpoint-100000/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef0a66a93651c55490a7f069ec878a68d27db46a11e2222541fe9c6ded602cf1
+size 16325

checkpoint-100000/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:705636c2daeef82bfeea12acec4d335d3b07a026ce57d3d3e0db4e9dc2b2bb17
+size 16325

checkpoint-100000/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d37c80b222aa5f5f25888cce288b804fcda67f69b6d390d167ba9e8226ed2fb
+size 16325

checkpoint-100000/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:891d0ff333b1113d2278c595ff57245737f23f5f620aad19cd2e3e3cf6d0ea7c
+size 16325

checkpoint-100000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5d226e068a64fdb5946e25b7b2e0d0f2e89f6c3c2fb105ea0567d6133bf383a
+size 1465

checkpoint-100000/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}