Hugging Face's logo

tachiwin
/
tokenizer_test

tokenizers
tokenizer
multilingual
byte-level-bpe
indigenous-languages
tachiwin
Model card Files
xet
 Community
ljcamargo commited on
Commit
65d14df
·
verified ·
1 Parent(s): 2e2eada

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +101 -0
tokenizer_config.json ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "extra_special_tokens": [
6
+ "<|im_start|>",
7
+ "<|im_end|>",
8
+ "<|tool_call_start|>",
9
+ "<|tool_call_end|>",
10
+ "<|tool_response_start|>",
11
+ "<|tool_response_end|>",
12
+ "<|reasoning_start|>",
13
+ "<|reasoning_end|>",
14
+ "<|fim_prefix|>",
15
+ "<|fim_middle|>",
16
+ "<|fim_suffix|>",
17
+ "<|fim_pad|>",
18
+ "<|repo_name|>",
19
+ "<|file_sep|>",
20
+ "<|lang:all|>",
21
+ "<|lang:amu|>",
22
+ "<|lang:amu0|>",
23
+ "<|lang:azg|>",
24
+ "<|lang:azn0|>",
25
+ "<|lang:azz|>",
26
+ "<|lang:cco|>",
27
+ "<|lang:cco0|>",
28
+ "<|lang:chd|>",
29
+ "<|lang:chf|>",
30
+ "<|lang:chq|>",
31
+ "<|lang:ctp0|>",
32
+ "<|lang:ctu|>",
33
+ "<|lang:cux|>",
34
+ "<|lang:de|>",
35
+ "<|lang:en-US|>",
36
+ "<|lang:eng|>",
37
+ "<|lang:es-MX|>",
38
+ "<|lang:fr|>",
39
+ "<|lang:hch|>",
40
+ "<|lang:hus|>",
41
+ "<|lang:huv|>",
42
+ "<|lang:lac|>",
43
+ "<|lang:mam|>",
44
+ "<|lang:mau|>",
45
+ "<|lang:maz|>",
46
+ "<|lang:meh|>",
47
+ "<|lang:mfy|>",
48
+ "<|lang:mix|>",
49
+ "<|lang:mmc|>",
50
+ "<|lang:nch|>",
51
+ "<|lang:ntp|>",
52
+ "<|lang:ocu|>",
53
+ "<|lang:ots|>",
54
+ "<|lang:pei|>",
55
+ "<|lang:pia|>",
56
+ "<|lang:pmq|>",
57
+ "<|lang:poi|>",
58
+ "<|lang:pt-BR|>",
59
+ "<|lang:pua|>",
60
+ "<|lang:quc|>",
61
+ "<|lang:sei|>",
62
+ "<|lang:spa|>",
63
+ "<|lang:tar0|>",
64
+ "<|lang:toj|>",
65
+ "<|lang:top|>",
66
+ "<|lang:tpp|>",
67
+ "<|lang:tpx|>",
68
+ "<|lang:trs|>",
69
+ "<|lang:tsz|>",
70
+ "<|lang:tzh|>",
71
+ "<|lang:tzo|>",
72
+ "<|lang:unk|>",
73
+ "<|lang:var|>",
74
+ "<|lang:yaq|>",
75
+ "<|lang:yua|>",
76
+ "<|lang:zai|>",
77
+ "<|lang:zam|>",
78
+ "<|lang:zoc|>",
79
+ "<|lang:zoq|>",
80
+ "<|reserved_0|>",
81
+ "<|reserved_1|>",
82
+ "<|reserved_2|>",
83
+ "<|reserved_3|>",
84
+ "<|reserved_4|>",
85
+ "<|reserved_5|>",
86
+ "<|reserved_6|>",
87
+ "<|reserved_7|>",
88
+ "<|reserved_8|>",
89
+ "<|reserved_9|>",
90
+ "<|reserved_10|>",
91
+ "<|reserved_11|>",
92
+ "<|reserved_12|>",
93
+ "<|reserved_13|>",
94
+ "<|reserved_14|>",
95
+ "<|reserved_15|>"
96
+ ],
97
+ "model_max_length": 4096,
98
+ "pad_token": "<|pad|>",
99
+ "tokenizer_class": "TokenizersBackend",
100
+ "unk_token": "<|unk|>"
101
+ }