Commit ·
e0543ab
1
Parent(s): 02d3e22
Remove unnused token
Browse files- special_tokens_map.json +0 -1
- tokenizer.json +5 -12
special_tokens_map.json
CHANGED
|
@@ -14,7 +14,6 @@
|
|
| 14 |
"<start_system>",
|
| 15 |
"<start_user>",
|
| 16 |
"<start_assistant>",
|
| 17 |
-
"<start_example>",
|
| 18 |
"<end_message>"
|
| 19 |
],
|
| 20 |
"eos_token": "<|endoftext|>"
|
|
|
|
| 14 |
"<start_system>",
|
| 15 |
"<start_user>",
|
| 16 |
"<start_assistant>",
|
|
|
|
| 17 |
"<end_message>"
|
| 18 |
],
|
| 19 |
"eos_token": "<|endoftext|>"
|
tokenizer.json
CHANGED
|
@@ -141,15 +141,6 @@
|
|
| 141 |
{
|
| 142 |
"id": 65020,
|
| 143 |
"special": true,
|
| 144 |
-
"content": "<start_example>",
|
| 145 |
-
"single_word": false,
|
| 146 |
-
"lstrip": false,
|
| 147 |
-
"rstrip": false,
|
| 148 |
-
"normalized": false
|
| 149 |
-
},
|
| 150 |
-
{
|
| 151 |
-
"id": 65019,
|
| 152 |
-
"special": true,
|
| 153 |
"content": "<end_message>",
|
| 154 |
"single_word": false,
|
| 155 |
"lstrip": false,
|
|
@@ -65217,8 +65208,8 @@
|
|
| 65217 |
"vee": 65016,
|
| 65218 |
"ĠSEN": 65017,
|
| 65219 |
"adero": 65018,
|
| 65220 |
-
"
|
| 65221 |
-
"<
|
| 65222 |
"<start_assistant>": 65021,
|
| 65223 |
"<start_user>": 65022,
|
| 65224 |
"<start_system>": 65023
|
|
@@ -130001,7 +129992,9 @@
|
|
| 130001 |
"s now",
|
| 130002 |
"Ġd end",
|
| 130003 |
"ve e",
|
| 130004 |
-
"ĠS EN"
|
|
|
|
|
|
|
| 130005 |
]
|
| 130006 |
}
|
| 130007 |
}
|
|
|
|
| 141 |
{
|
| 142 |
"id": 65020,
|
| 143 |
"special": true,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
"content": "<end_message>",
|
| 145 |
"single_word": false,
|
| 146 |
"lstrip": false,
|
|
|
|
| 65208 |
"vee": 65016,
|
| 65209 |
"ĠSEN": 65017,
|
| 65210 |
"adero": 65018,
|
| 65211 |
+
"ĠAFF": 65019,
|
| 65212 |
+
"<end_message>": 65020,
|
| 65213 |
"<start_assistant>": 65021,
|
| 65214 |
"<start_user>": 65022,
|
| 65215 |
"<start_system>": 65023
|
|
|
|
| 129992 |
"s now",
|
| 129993 |
"Ġd end",
|
| 129994 |
"ve e",
|
| 129995 |
+
"ĠS EN",
|
| 129996 |
+
"ad ero",
|
| 129997 |
+
"ĠA FF"
|
| 129998 |
]
|
| 129999 |
}
|
| 130000 |
}
|