File size: 425 Bytes
887c99e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
{
  "tokenizer_class": "PreTrainedTokenizerFast",
  "special_tokens_map": {
    "bos_token": "<|start_header_id|>",
    "eos_token": "<|eot_id|>",
    "additional_special_tokens": [
      "<|start_header_id|>",
      "<|end_header_id|>",
      "<|start_header_id|>system<|end_header_id|>",
      "<|start_header_id|>assistant<|end_header_id|>",
      "<|start_header_id|>user<|end_header_id|>",
      "<|eot_id|>"
    ]
  }
}