Upload processor

added_tokens.json

@@ -1518,6 +1518,7 @@
   "<|da|>": 50285,
   "<|de|>": 50261,
   "<|el|>": 50281,
+  "<|endoftext|>": 50257,
   "<|en|>": 50259,
   "<|es|>": 50262,
   "<|et|>": 50307,

special_tokens_map.json

@@ -108,26 +108,8 @@
     "<|nocaptions|>",
     "<|notimestamps|>"
   ],
-  "bos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
   "pad_token": "<|endoftext|>",
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "unk_token": "<|endoftext|>"
 }