Upload tokenizer
Browse files- tokenizer.json +1 -1
tokenizer.json
CHANGED
|
@@ -68,7 +68,7 @@
|
|
| 68 |
{
|
| 69 |
"type": "Split",
|
| 70 |
"pattern": {
|
| 71 |
-
"Regex": "(\\[[a-z]+]|\\[[A-Z][a-z]?|Br?|Cl?|N|O|S|P|F|I|H[2-
|
| 72 |
},
|
| 73 |
"behavior": "Isolated",
|
| 74 |
"invert": false
|
|
|
|
| 68 |
{
|
| 69 |
"type": "Split",
|
| 70 |
"pattern": {
|
| 71 |
+
"Regex": "(\\[[a-z]+]|\\[[A-Z][a-z]?|Br?|Cl?|N|O|S|P|F|I|H[2-4]?|\\[|\\]|,|;|\\(|\\)|\\.|=|\\#|-|\\+|\\\\|/|:|~|@|\\?|>|\\*|\\$|%|(?<=%)[0-9]{2}|(?<=,)[0-9]{2}(?=\\))|(?<=\\()[0-9]{2}(?=,)|(?<=:)[0-9]{2}(?=\\])|[0-9]|\\|)"
|
| 72 |
},
|
| 73 |
"behavior": "Isolated",
|
| 74 |
"invert": false
|