KorByte-128K / tokenizer_config.json
dawncr0w's picture
Publish KorByte-128K tokenizer and reproducibility evidence
b3c2a26 verified
Raw
History Blame Contribute Delete
6.35 kB
{
"add_bos_token": false,
"add_eos_token": false,
"additional_special_tokens": [
"<|system|>",
"<|user|>",
"<|assistant|>",
"<|tool|>",
"<|tool_result|>",
"<|document_start|>",
"<|document_end|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|reserved_000|>",
"<|reserved_001|>",
"<|reserved_002|>",
"<|reserved_003|>",
"<|reserved_004|>",
"<|reserved_005|>",
"<|reserved_006|>",
"<|reserved_007|>",
"<|reserved_008|>",
"<|reserved_009|>",
"<|reserved_010|>",
"<|reserved_011|>",
"<|reserved_012|>",
"<|reserved_013|>",
"<|reserved_014|>",
"<|reserved_015|>",
"<|reserved_016|>",
"<|reserved_017|>",
"<|reserved_018|>",
"<|reserved_019|>",
"<|reserved_020|>",
"<|reserved_021|>",
"<|reserved_022|>",
"<|reserved_023|>",
"<|reserved_024|>",
"<|reserved_025|>",
"<|reserved_026|>",
"<|reserved_027|>",
"<|reserved_028|>",
"<|reserved_029|>",
"<|reserved_030|>",
"<|reserved_031|>",
"<|reserved_032|>",
"<|reserved_033|>",
"<|reserved_034|>",
"<|reserved_035|>",
"<|reserved_036|>",
"<|reserved_037|>",
"<|reserved_038|>",
"<|reserved_039|>",
"<|reserved_040|>",
"<|reserved_041|>",
"<|reserved_042|>",
"<|reserved_043|>",
"<|reserved_044|>",
"<|reserved_045|>",
"<|reserved_046|>",
"<|reserved_047|>",
"<|reserved_048|>",
"<|reserved_049|>",
"<|reserved_050|>",
"<|reserved_051|>",
"<|reserved_052|>",
"<|reserved_053|>",
"<|reserved_054|>",
"<|reserved_055|>",
"<|reserved_056|>",
"<|reserved_057|>",
"<|reserved_058|>",
"<|reserved_059|>",
"<|reserved_060|>",
"<|reserved_061|>",
"<|reserved_062|>",
"<|reserved_063|>",
"<|reserved_064|>",
"<|reserved_065|>",
"<|reserved_066|>",
"<|reserved_067|>",
"<|reserved_068|>",
"<|reserved_069|>",
"<|reserved_070|>",
"<|reserved_071|>",
"<|reserved_072|>",
"<|reserved_073|>",
"<|reserved_074|>",
"<|reserved_075|>",
"<|reserved_076|>",
"<|reserved_077|>",
"<|reserved_078|>",
"<|reserved_079|>",
"<|reserved_080|>",
"<|reserved_081|>",
"<|reserved_082|>",
"<|reserved_083|>",
"<|reserved_084|>",
"<|reserved_085|>",
"<|reserved_086|>",
"<|reserved_087|>",
"<|reserved_088|>",
"<|reserved_089|>",
"<|reserved_090|>",
"<|reserved_091|>",
"<|reserved_092|>",
"<|reserved_093|>",
"<|reserved_094|>",
"<|reserved_095|>",
"<|reserved_096|>",
"<|reserved_097|>",
"<|reserved_098|>",
"<|reserved_099|>",
"<|reserved_100|>",
"<|reserved_101|>",
"<|reserved_102|>",
"<|reserved_103|>",
"<|reserved_104|>",
"<|reserved_105|>",
"<|reserved_106|>",
"<|reserved_107|>",
"<|reserved_108|>",
"<|reserved_109|>",
"<|reserved_110|>",
"<|reserved_111|>",
"<|reserved_112|>",
"<|reserved_113|>",
"<|reserved_114|>",
"<|reserved_115|>",
"<|reserved_116|>",
"<|reserved_117|>",
"<|reserved_118|>",
"<|reserved_119|>",
"<|reserved_120|>",
"<|reserved_121|>",
"<|reserved_122|>",
"<|reserved_123|>",
"<|reserved_124|>",
"<|reserved_125|>",
"<|reserved_126|>",
"<|reserved_127|>",
"<|reserved_128|>",
"<|reserved_129|>",
"<|reserved_130|>",
"<|reserved_131|>",
"<|reserved_132|>",
"<|reserved_133|>",
"<|reserved_134|>",
"<|reserved_135|>",
"<|reserved_136|>",
"<|reserved_137|>",
"<|reserved_138|>",
"<|reserved_139|>",
"<|reserved_140|>",
"<|reserved_141|>",
"<|reserved_142|>",
"<|reserved_143|>",
"<|reserved_144|>",
"<|reserved_145|>",
"<|reserved_146|>",
"<|reserved_147|>",
"<|reserved_148|>",
"<|reserved_149|>",
"<|reserved_150|>",
"<|reserved_151|>",
"<|reserved_152|>",
"<|reserved_153|>",
"<|reserved_154|>",
"<|reserved_155|>",
"<|reserved_156|>",
"<|reserved_157|>",
"<|reserved_158|>",
"<|reserved_159|>",
"<|reserved_160|>",
"<|reserved_161|>",
"<|reserved_162|>",
"<|reserved_163|>",
"<|reserved_164|>",
"<|reserved_165|>",
"<|reserved_166|>",
"<|reserved_167|>",
"<|reserved_168|>",
"<|reserved_169|>",
"<|reserved_170|>",
"<|reserved_171|>",
"<|reserved_172|>",
"<|reserved_173|>",
"<|reserved_174|>",
"<|reserved_175|>",
"<|reserved_176|>",
"<|reserved_177|>",
"<|reserved_178|>",
"<|reserved_179|>",
"<|reserved_180|>",
"<|reserved_181|>",
"<|reserved_182|>",
"<|reserved_183|>",
"<|reserved_184|>",
"<|reserved_185|>",
"<|reserved_186|>",
"<|reserved_187|>",
"<|reserved_188|>",
"<|reserved_189|>",
"<|reserved_190|>",
"<|reserved_191|>",
"<|reserved_192|>",
"<|reserved_193|>",
"<|reserved_194|>",
"<|reserved_195|>",
"<|reserved_196|>",
"<|reserved_197|>",
"<|reserved_198|>",
"<|reserved_199|>",
"<|reserved_200|>",
"<|reserved_201|>",
"<|reserved_202|>",
"<|reserved_203|>",
"<|reserved_204|>",
"<|reserved_205|>",
"<|reserved_206|>",
"<|reserved_207|>",
"<|reserved_208|>",
"<|reserved_209|>",
"<|reserved_210|>",
"<|reserved_211|>",
"<|reserved_212|>",
"<|reserved_213|>",
"<|reserved_214|>",
"<|reserved_215|>",
"<|reserved_216|>",
"<|reserved_217|>",
"<|reserved_218|>",
"<|reserved_219|>",
"<|reserved_220|>",
"<|reserved_221|>",
"<|reserved_222|>",
"<|reserved_223|>",
"<|reserved_224|>",
"<|reserved_225|>",
"<|reserved_226|>",
"<|reserved_227|>",
"<|reserved_228|>",
"<|reserved_229|>",
"<|reserved_230|>",
"<|reserved_231|>",
"<|reserved_232|>",
"<|reserved_233|>",
"<|reserved_234|>",
"<|reserved_235|>",
"<|reserved_236|>",
"<|reserved_237|>",
"<|reserved_238|>",
"<|reserved_239|>",
"<|reserved_240|>",
"<|reserved_241|>"
],
"bos_token": "<|begin_of_text|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|end_of_text|>",
"model_max_length": 32768,
"pad_token": "<|pad|>",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|unk|>"
}