Update tokenizer.json
#6
by
iamPi
- opened
- tokenizer.json +1 -4
tokenizer.json
CHANGED
|
@@ -80,10 +80,7 @@
|
|
| 80 |
"9": 9,
|
| 81 |
"Ċ": 10,
|
| 82 |
"user": 11,
|
| 83 |
-
"assistant": 12
|
| 84 |
-
"<|endoftext|>": 13,
|
| 85 |
-
"<|im_start|>": 14,
|
| 86 |
-
"<|im_end|>": 15
|
| 87 |
},
|
| 88 |
"unk_token": "<|endoftext|>"
|
| 89 |
}
|
|
|
|
| 80 |
"9": 9,
|
| 81 |
"Ċ": 10,
|
| 82 |
"user": 11,
|
| 83 |
+
"assistant": 12
|
|
|
|
|
|
|
|
|
|
| 84 |
},
|
| 85 |
"unk_token": "<|endoftext|>"
|
| 86 |
}
|