Update tokenizer.json
Browse files- tokenizer.json +0 -28
tokenizer.json
CHANGED
|
@@ -1125,34 +1125,6 @@
|
|
| 1125 |
"每",
|
| 1126 |
0.0
|
| 1127 |
],
|
| 1128 |
-
[
|
| 1129 |
-
"<bos>",
|
| 1130 |
-
0.0
|
| 1131 |
-
],
|
| 1132 |
-
[
|
| 1133 |
-
"<end_of_turn>",
|
| 1134 |
-
0.0
|
| 1135 |
-
],
|
| 1136 |
-
[
|
| 1137 |
-
"<eos>",
|
| 1138 |
-
0.0
|
| 1139 |
-
],
|
| 1140 |
-
[
|
| 1141 |
-
"<pad>",
|
| 1142 |
-
0.0
|
| 1143 |
-
],
|
| 1144 |
-
[
|
| 1145 |
-
"<start_of_turn>",
|
| 1146 |
-
0.0
|
| 1147 |
-
],
|
| 1148 |
-
[
|
| 1149 |
-
"model",
|
| 1150 |
-
0.0
|
| 1151 |
-
],
|
| 1152 |
-
[
|
| 1153 |
-
"user",
|
| 1154 |
-
0.0
|
| 1155 |
-
]
|
| 1156 |
],
|
| 1157 |
"byte_fallback": false
|
| 1158 |
}
|
|
|
|
| 1125 |
"每",
|
| 1126 |
0.0
|
| 1127 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1128 |
],
|
| 1129 |
"byte_fallback": false
|
| 1130 |
}
|