Fix the arguments
Browse files- tokenizer.json +41 -3
tokenizer.json
CHANGED
|
@@ -216,11 +216,49 @@
|
|
| 216 |
"post_processor": {
|
| 217 |
"type": "TemplateProcessing",
|
| 218 |
"single": [
|
| 219 |
-
|
| 220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
],
|
| 222 |
"special_tokens": {
|
| 223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
}
|
| 225 |
},
|
| 226 |
"decoder": {
|
|
|
|
| 216 |
"post_processor": {
|
| 217 |
"type": "TemplateProcessing",
|
| 218 |
"single": [
|
| 219 |
+
{
|
| 220 |
+
"SpecialToken": {
|
| 221 |
+
"id": "<|begin_of_text|>",
|
| 222 |
+
"type_id": 0
|
| 223 |
+
}
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"Sequence": {
|
| 227 |
+
"id": "A",
|
| 228 |
+
"type_id": 0
|
| 229 |
+
}
|
| 230 |
+
}
|
| 231 |
+
],
|
| 232 |
+
"pair": [
|
| 233 |
+
{
|
| 234 |
+
"SpecialToken": {
|
| 235 |
+
"id": "<|begin_of_text|>",
|
| 236 |
+
"type_id": 0
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"Sequence": {
|
| 241 |
+
"id": "A",
|
| 242 |
+
"type_id": 0
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"Sequence": {
|
| 247 |
+
"id": "B",
|
| 248 |
+
"type_id": 1
|
| 249 |
+
}
|
| 250 |
+
}
|
| 251 |
],
|
| 252 |
"special_tokens": {
|
| 253 |
+
"<|begin_of_text|>": {
|
| 254 |
+
"id": "<|begin_of_text|>",
|
| 255 |
+
"ids": [
|
| 256 |
+
100256
|
| 257 |
+
],
|
| 258 |
+
"tokens": [
|
| 259 |
+
"<|begin_of_text|>"
|
| 260 |
+
]
|
| 261 |
+
}
|
| 262 |
}
|
| 263 |
},
|
| 264 |
"decoder": {
|