623 / tokenizer /tokenizer_config.json
ffdddf's picture
Upload 9 files
60d1dcb verified
raw
history blame
349 Bytes
{
"model_type": "clip",
"tokenizer_type": "CLIPTokenizer",
"vocab_size": 49408,
"max_position_embeddings": 77,
"pad_token_id": 1,
"bos_token_id": 0,
"eos_token_id": 2,
"special_tokens": {
"pad_token": "<|pad|>",
"bos_token": "<|startoftext|>",
"eos_token": "<|endoftext|>"
}
}