Set `<start_of_turn` and `<end_of_turn>` to be special tokens (#2)
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +2 -2
tokenizer.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9cf8cddb82b68ab495546855952d707acfa17f23c97031e0cf346b207d73faae
|
| 3 |
+
size 17525357
|
tokenizer_config.json
CHANGED
|
@@ -856,7 +856,7 @@
|
|
| 856 |
"normalized": false,
|
| 857 |
"rstrip": false,
|
| 858 |
"single_word": false,
|
| 859 |
-
"special":
|
| 860 |
},
|
| 861 |
"107": {
|
| 862 |
"content": "<end_of_turn>",
|
|
@@ -864,7 +864,7 @@
|
|
| 864 |
"normalized": false,
|
| 865 |
"rstrip": false,
|
| 866 |
"single_word": false,
|
| 867 |
-
"special":
|
| 868 |
},
|
| 869 |
"108": {
|
| 870 |
"content": "\n",
|
|
|
|
| 856 |
"normalized": false,
|
| 857 |
"rstrip": false,
|
| 858 |
"single_word": false,
|
| 859 |
+
"special": true
|
| 860 |
},
|
| 861 |
"107": {
|
| 862 |
"content": "<end_of_turn>",
|
|
|
|
| 864 |
"normalized": false,
|
| 865 |
"rstrip": false,
|
| 866 |
"single_word": false,
|
| 867 |
+
"special": true
|
| 868 |
},
|
| 869 |
"108": {
|
| 870 |
"content": "\n",
|