21930db cf6447e 21930db
1
2
3
4
5
6
7
8
9
{ "backend": "tokenizers", "bos_token": "<s>", "eos_token": "<|im_end|>", "model_max_length": 4096, "pad_token": "<pad>", "tokenizer_class": "PreTrainedTokenizerFast" }
