Hugging Face's logo

loubnabnl
/
starcoder_tokenizer

Model card Files
xet
Community
starcoder_tokenizer
File size: 734 Bytes
54feb14
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
{
    "add_prefix_space": false,
    "additional_special_tokens": [
      "<|endoftext|>",
      "<fim_prefix>",
      "<fim_middle>",
      "<fim_suffix>",
      "<fim_pad>",
      "<filename>",
      "<gh_stars>",
      "<issue_start>",
      "<issue_comment>",
      "<issue_closed>",
      "<jupyter_start>",
      "<jupyter_text>",
      "<jupyter_code>",
      "<jupyter_output>",
      "<empty_output>",
      "<commit_before>",
      "<commit_msg>",
      "<commit_after>",
      "<reponame>"
    ],
    "bos_token": "<|endoftext|>",
    "eos_token": "<|endoftext|>",
    "model_max_length": 1000000000000000019884624838656,
    "tokenizer_class": "GPT2Tokenizer",
    "unk_token": "<|endoftext|>",
    "vocab_size": 49152
  }