update config and vocab with new special tokens handling

config.json

@@ -1,36 +1,39 @@
 {
-  "data": null,
-  "tgt_vocab_size": 128256,
+  "eos_token": "<|end_of_text|>",
+  "share_vocab": true,
   "transforms": [
     "onmt_tokenize",
     "filtertoolong"
   ],
-  "src_vocab": null,
-  "decoder_start_token": "<s>",
-  "skip_empty_level": "silent",
-  "share_vocab": true,
-  "n_sample": 0,
-  "vocab_size_multiple": 8,
+  "data": null,
+  "tgt_vocab_size": 128256,
   "save_data": null,
+  "vocab_size_multiple": 8,
   "tgt_vocab": null,
+  "n_sample": 0,
+  "src_vocab": null,
+  "skip_empty_level": "silent",
+  "bos_token": "<|begin_of_text|>",
   "src_vocab_size": 128256,
+  "decoder_start_token": "<|begin_of_text|>",
+  "pad_token": "<blank>",
   "training": {
-    "accum_count": [
-      32
-    ],
-    "batch_size": 896,
     "accum_steps": [
       0
     ],
-    "group_size": 0,
-    "valid_batch_size": 256,
     "batch_size_multiple": 1,
-    "normalization": "tokens",
+    "valid_batch_size": 256,
+    "batch_type": "tokens",
     "quant_type": "",
+    "accum_count": [
+      32
+    ],
     "w_bit": 0,
     "compute_dtype": "torch.bfloat16",
-    "batch_type": "tokens",
-    "quant_layers": []
+    "normalization": "tokens",
+    "quant_layers": [],
+    "group_size": 0,
+    "batch_size": 896
   },
   "transforms_configs": {
     "filtertoolong": {
@@ -1070,42 +1073,42 @@
     }
   },
   "model": {
-    "num_experts_per_tok": 0,
-    "add_ffnbias": false,
-    "left_pad": true,
-    "layer_norm": "rms",
+    "head_dim": null,
+    "sliding_window": 0,
+    "architecture": "transformer_lm",
     "num_experts": 0,
     "heads": 32,
-    "parallel_residual": false,
-    "add_qkvbias": false,
-    "head_dim": null,
-    "transformer_ff": 14336,
-    "heads_kv": 8,
+    "hidden_size": 4096,
     "norm_eps": 1e-05,
-    "architecture": "transformer_lm",
+    "num_experts_per_tok": 0,
     "mlp_activation_fn": "gated-silu",
-    "hidden_size": 4096,
-    "shared_layer_norm": false,
-    "sliding_window": 0,
+    "left_pad": true,
     "layers": 32,
+    "layer_norm": "rms",
+    "shared_layer_norm": false,
+    "add_ffnbias": false,
+    "heads_kv": 8,
+    "add_qkvbias": false,
+    "parallel_residual": false,
+    "transformer_ff": 14336,
     "embeddings": {
       "n_positions": 0,
+      "tgt_word_vec_size": 4096,
       "position_encoding_type": "Rotary",
-      "src_word_vec_size": 4096,
-      "tgt_word_vec_size": 4096
+      "src_word_vec_size": 4096
     },
     "rope_config": {
-      "rotary_theta": 500000,
-      "rotary_interleave": false
+      "rotary_interleave": false,
+      "rotary_theta": 500000
     },
     "decoder": {
       "position_encoding_type": "Rotary",
-      "decoder_type": "transformer_lm",
       "n_positions": 0,
+      "decoder_type": "transformer_lm",
       "tgt_word_vec_size": 4096,
       "rope_config": {
-        "rotary_theta": 500000,
-        "rotary_interleave": false
+        "rotary_interleave": false,
+        "rotary_theta": 500000
       }
     }
   },

vocab.json

@@ -128000,7 +128000,7 @@
     "ĠвÑĭÑģокой",
     "ãĥ¼ãĥ¼",
     "éĶ¦",
-    "<s>",
+    "<|begin_of_text|>",
     "<|end_of_text|>",
     "<|reserved_special_token_0|>",
     "<|reserved_special_token_1|>",
@@ -128009,7 +128009,7 @@
     "<|start_header_id|>",
     "<|end_header_id|>",
     "<|reserved_special_token_4|>",
-    "</s>",
+    "<|eot_id|>",
     "<|reserved_special_token_5|>",
     "<|reserved_special_token_6|>",
     "<|reserved_special_token_7|>",
@@ -256258,7 +256258,7 @@
     "ĠвÑĭÑģокой",
     "ãĥ¼ãĥ¼",
     "éĶ¦",
-    "<s>",
+    "<|begin_of_text|>",
     "<|end_of_text|>",
     "<|reserved_special_token_0|>",
     "<|reserved_special_token_1|>",
@@ -256267,7 +256267,7 @@
     "<|start_header_id|>",
     "<|end_header_id|>",
     "<|reserved_special_token_4|>",
-    "</s>",
+    "<|eot_id|>",
     "<|reserved_special_token_5|>",
     "<|reserved_special_token_6|>",
     "<|reserved_special_token_7|>",
@@ -256515,5 +256515,10 @@
     "<|reserved_special_token_249|>",
     "<|reserved_special_token_250|>"
   ],
-  "decoder_start_token": "<s>"
+  "decoder_start_token": "<|begin_of_text|>",
+  "specials": {
+    "bos_token": "<|begin_of_text|>",
+    "eos_token": "<|end_of_text|>",
+    "pad_token": "<blank>"
+  }
 }