Transformers
Volowan commited on
Commit
69aff18
·
verified ·
1 Parent(s): c4934f2

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -1
tokenizer.json CHANGED
@@ -68,7 +68,7 @@
68
  {
69
  "type": "Split",
70
  "pattern": {
71
- "Regex": "(\\[[a-z]+]|\\[[A-Z][a-z]?|Br?|Cl?|N|O|S|P|F|I|H[2-3]?|\\[|\\]|,|;|\\(|\\)|\\.|=|\\#|-|\\+|\\\\|/|:|~|@|\\?|>|\\*|\\$|%|(?<=%)[0-9]{2}|(?<=,)[0-9]{2}(?=\\))|(?<=\\()[0-9]{2}(?=,)|(?<=:)[0-9]{2}(?=\\])|[0-9]|\\|)"
72
  },
73
  "behavior": "Isolated",
74
  "invert": false
 
68
  {
69
  "type": "Split",
70
  "pattern": {
71
+ "Regex": "(\\[[a-z]+]|\\[[A-Z][a-z]?|Br?|Cl?|N|O|S|P|F|I|H[2-4]?|\\[|\\]|,|;|\\(|\\)|\\.|=|\\#|-|\\+|\\\\|/|:|~|@|\\?|>|\\*|\\$|%|(?<=%)[0-9]{2}|(?<=,)[0-9]{2}(?=\\))|(?<=\\()[0-9]{2}(?=,)|(?<=:)[0-9]{2}(?=\\])|[0-9]|\\|)"
72
  },
73
  "behavior": "Isolated",
74
  "invert": false