Set [SEP] as EOS token: update eos_token, eos_token_id, and generation config

config.json

@@ -6,7 +6,7 @@
   "attn_pdrop": 0.1,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
-  "eos_token_id": 50256,
+  "eos_token_id": 2,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt2",

generation_config.json

@@ -1,7 +1,7 @@
 {
   "_from_model_config": true,
   "bos_token_id": 50256,
-  "eos_token_id": 30001,
+  "eos_token_id": 2,
   "pad_token_id": 3,
   "transformers_version": "4.55.4"
 }

special_tokens_map.json

@@ -7,7 +7,7 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "</s>",
+    "content": "[SEP]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer_config.json

@@ -59,7 +59,7 @@
   },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
+  "eos_token": "[SEP]",
   "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",