Hugging Face's logo

davaa33
/
mns_tokenizer

Transformers
Model card Files
xet
Community
davaa33 commited on
Commit
8a7a1c8
·
verified ·
1 Parent(s): 1bbb5d8

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +23 -2
  2. spiece.model +2 -2
  3. tokenizer_config.json +19 -172
special_tokens_map.json CHANGED
@@ -1,18 +1,32 @@
1
  {
2
  "bos_token": {
3
- "content": "<s>",
 
 
 
 
 
 
 
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "</s>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
 
 
 
 
 
 
 
16
  "pad_token": {
17
  "content": "<pad>",
18
  "lstrip": false,
@@ -20,6 +34,13 @@
20
  "rstrip": false,
21
  "single_word": false
22
  },
 
 
 
 
 
 
 
23
  "unk_token": {
24
  "content": "<unk>",
25
  "lstrip": false,
 
1
  {
2
  "bos_token": {
3
+ "content": "[CLS]",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "cls_token": {
10
+ "content": "[CLS]",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
  "eos_token": {
17
+ "content": "[SEP]",
18
  "lstrip": false,
19
  "normalized": false,
20
  "rstrip": false,
21
  "single_word": false
22
  },
23
+ "mask_token": {
24
+ "content": "[MASK]",
25
+ "lstrip": true,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
  "pad_token": {
31
  "content": "<pad>",
32
  "lstrip": false,
 
34
  "rstrip": false,
35
  "single_word": false
36
  },
37
+ "sep_token": {
38
+ "content": "[SEP]",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
  "unk_token": {
45
  "content": "<unk>",
46
  "lstrip": false,
spiece.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a257a7769baa23c0f127b1497974b34e557d0346e6b00bc126ab1292566b0d2c
3
- size 478498
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f651a67478f78aa17f004fb7b70ec4539fda8d1e2cca81f26a9f0c2a8e4579c1
3
+ size 461869
tokenizer_config.json CHANGED
@@ -8,23 +8,23 @@
8
  "single_word": false,
9
  "special": true
10
  },
11
- "1": {
12
- "content": "<s>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
  "special": true
18
  },
19
- "2": {
20
- "content": "</s>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
26
  },
27
- "8000": {
28
  "content": "<pad>",
29
  "lstrip": false,
30
  "normalized": false,
@@ -32,181 +32,28 @@
32
  "single_word": false,
33
  "special": true
34
  },
35
- "8001": {
36
- "content": "–",
37
- "lstrip": false,
38
- "normalized": true,
39
- "rstrip": false,
40
- "single_word": false,
41
- "special": false
42
- },
43
- "8002": {
44
- "content": "'",
45
- "lstrip": false,
46
- "normalized": true,
47
- "rstrip": false,
48
- "single_word": false,
49
- "special": false
50
- },
51
  "8003": {
52
- "content": "`",
53
- "lstrip": false,
54
- "normalized": true,
55
- "rstrip": false,
56
- "single_word": false,
57
- "special": false
58
- },
59
- "8004": {
60
- "content": "\\",
61
- "lstrip": false,
62
- "normalized": true,
63
- "rstrip": false,
64
- "single_word": false,
65
- "special": false
66
- },
67
- "8005": {
68
- "content": "#",
69
- "lstrip": false,
70
- "normalized": true,
71
- "rstrip": false,
72
- "single_word": false,
73
- "special": false
74
- },
75
- "8006": {
76
- "content": "*",
77
- "lstrip": false,
78
- "normalized": true,
79
- "rstrip": false,
80
- "single_word": false,
81
- "special": false
82
- },
83
- "8007": {
84
- "content": "<",
85
- "lstrip": false,
86
- "normalized": true,
87
- "rstrip": false,
88
- "single_word": false,
89
- "special": false
90
- },
91
- "8008": {
92
- "content": ">",
93
- "lstrip": false,
94
- "normalized": true,
95
- "rstrip": false,
96
- "single_word": false,
97
- "special": false
98
- },
99
- "8009": {
100
- "content": "…",
101
- "lstrip": false,
102
- "normalized": true,
103
- "rstrip": false,
104
- "single_word": false,
105
- "special": false
106
- },
107
- "8010": {
108
- "content": "§",
109
- "lstrip": false,
110
- "normalized": true,
111
- "rstrip": false,
112
- "single_word": false,
113
- "special": false
114
- },
115
- "8011": {
116
- "content": "¶",
117
- "lstrip": false,
118
- "normalized": true,
119
- "rstrip": false,
120
- "single_word": false,
121
- "special": false
122
- },
123
- "8012": {
124
- "content": "᠐",
125
- "lstrip": false,
126
- "normalized": true,
127
- "rstrip": false,
128
- "single_word": false,
129
- "special": false
130
- },
131
- "8013": {
132
- "content": "᠑",
133
- "lstrip": false,
134
- "normalized": true,
135
- "rstrip": false,
136
- "single_word": false,
137
- "special": false
138
- },
139
- "8014": {
140
- "content": "᠒",
141
- "lstrip": false,
142
- "normalized": true,
143
- "rstrip": false,
144
- "single_word": false,
145
- "special": false
146
- },
147
- "8015": {
148
- "content": "᠓",
149
- "lstrip": false,
150
- "normalized": true,
151
- "rstrip": false,
152
- "single_word": false,
153
- "special": false
154
- },
155
- "8016": {
156
- "content": "᠔",
157
- "lstrip": false,
158
- "normalized": true,
159
- "rstrip": false,
160
- "single_word": false,
161
- "special": false
162
- },
163
- "8017": {
164
- "content": "᠕",
165
- "lstrip": false,
166
- "normalized": true,
167
- "rstrip": false,
168
- "single_word": false,
169
- "special": false
170
- },
171
- "8018": {
172
- "content": "᠖",
173
- "lstrip": false,
174
- "normalized": true,
175
- "rstrip": false,
176
- "single_word": false,
177
- "special": false
178
- },
179
- "8019": {
180
- "content": "᠗",
181
- "lstrip": false,
182
- "normalized": true,
183
- "rstrip": false,
184
- "single_word": false,
185
- "special": false
186
- },
187
- "8020": {
188
- "content": "᠘",
189
- "lstrip": false,
190
- "normalized": true,
191
- "rstrip": false,
192
- "single_word": false,
193
- "special": false
194
- },
195
- "8021": {
196
- "content": "᠙",
197
- "lstrip": false,
198
- "normalized": true,
199
  "rstrip": false,
200
  "single_word": false,
201
- "special": false
202
  }
203
  },
204
- "bos_token": "<s>",
205
  "clean_up_tokenization_spaces": false,
206
- "eos_token": "</s>",
 
 
207
  "extra_special_tokens": {},
 
 
208
  "model_max_length": 1000000000000000019884624838656,
209
  "pad_token": "<pad>",
210
- "tokenizer_class": "PreTrainedTokenizer",
 
 
 
211
  "unk_token": "<unk>"
212
  }
 
8
  "single_word": false,
9
  "special": true
10
  },
11
+ "8000": {
12
+ "content": "[CLS]",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
  "special": true
18
  },
19
+ "8001": {
20
+ "content": "[SEP]",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "8002": {
28
  "content": "<pad>",
29
  "lstrip": false,
30
  "normalized": false,
 
32
  "single_word": false,
33
  "special": true
34
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  "8003": {
36
+ "content": "[MASK]",
37
+ "lstrip": true,
38
+ "normalized": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  "rstrip": false,
40
  "single_word": false,
41
+ "special": true
42
  }
43
  },
44
+ "bos_token": "[CLS]",
45
  "clean_up_tokenization_spaces": false,
46
+ "cls_token": "[CLS]",
47
+ "do_lower_case": true,
48
+ "eos_token": "[SEP]",
49
  "extra_special_tokens": {},
50
+ "keep_accents": false,
51
+ "mask_token": "[MASK]",
52
  "model_max_length": 1000000000000000019884624838656,
53
  "pad_token": "<pad>",
54
+ "remove_space": true,
55
+ "sep_token": "[SEP]",
56
+ "sp_model_kwargs": {},
57
+ "tokenizer_class": "AlbertTokenizer",
58
  "unk_token": "<unk>"
59
  }