add DPS based tokens
Browse files- tokenizer.json +5 -5
tokenizer.json
CHANGED
|
@@ -64144,11 +64144,11 @@
|
|
| 64144 |
"댜": 63987,
|
| 64145 |
"<|im_start|>": 63988,
|
| 64146 |
"<|im_end|>": 63989,
|
| 64147 |
-
"<|
|
| 64148 |
-
"<|
|
| 64149 |
-
"<|
|
| 64150 |
-
"<|
|
| 64151 |
-
"<|
|
| 64152 |
"<|reserved_5|>": 63995,
|
| 64153 |
"<|reserved_6|>": 63996,
|
| 64154 |
"<|reserved_7|>": 63997,
|
|
|
|
| 64144 |
"댜": 63987,
|
| 64145 |
"<|im_start|>": 63988,
|
| 64146 |
"<|im_end|>": 63989,
|
| 64147 |
+
"<|sep|>": 63990,
|
| 64148 |
+
"<|endoftext|>": 63991,
|
| 64149 |
+
"<|acc|>": 63992,
|
| 64150 |
+
"<|rrn|>": 63993,
|
| 64151 |
+
"<|tel|>": 63994,
|
| 64152 |
"<|reserved_5|>": 63995,
|
| 64153 |
"<|reserved_6|>": 63996,
|
| 64154 |
"<|reserved_7|>": 63997,
|