bb4cfd4 c95b323 8d7edf6 bb4cfd4 edbe4f9
1
2
3
4
5
6
7
8
9
{ "bos_token": "<|startoftext|>", "clean_up_tokenization_spaces": true, "eos_token": "<|eos|>", "model_max_length": 262144, "pad_token": "<|pad|>", "tokenizer_class": "PreTrainedTokenizerFast" }
