Commit ·
2e254c4
1
Parent(s): a9cc2de
Training done
Browse files- added_tokens.json +2 -0
- tokenizer.json +18 -0
added_tokens.json
CHANGED
|
@@ -1,8 +1,10 @@
|
|
| 1 |
{
|
|
|
|
| 2 |
"<line>": 57531,
|
| 3 |
"<one>": 57525,
|
| 4 |
"<s_iitcdip>": 57523,
|
| 5 |
"<s_synthdog>": 57524,
|
|
|
|
| 6 |
"<sep/>": 57522,
|
| 7 |
"<vertical_bar>": 57532,
|
| 8 |
"<x_end>": 57528,
|
|
|
|
| 1 |
{
|
| 2 |
+
"<dot>": 57534,
|
| 3 |
"<line>": 57531,
|
| 4 |
"<one>": 57525,
|
| 5 |
"<s_iitcdip>": 57523,
|
| 6 |
"<s_synthdog>": 57524,
|
| 7 |
+
"<scatter>": 57533,
|
| 8 |
"<sep/>": 57522,
|
| 9 |
"<vertical_bar>": 57532,
|
| 10 |
"<x_end>": 57528,
|
tokenizer.json
CHANGED
|
@@ -146,6 +146,24 @@
|
|
| 146 |
"rstrip": false,
|
| 147 |
"normalized": true,
|
| 148 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
}
|
| 150 |
],
|
| 151 |
"normalizer": {
|
|
|
|
| 146 |
"rstrip": false,
|
| 147 |
"normalized": true,
|
| 148 |
"special": false
|
| 149 |
+
},
|
| 150 |
+
{
|
| 151 |
+
"id": 57533,
|
| 152 |
+
"content": "<scatter>",
|
| 153 |
+
"single_word": false,
|
| 154 |
+
"lstrip": false,
|
| 155 |
+
"rstrip": false,
|
| 156 |
+
"normalized": true,
|
| 157 |
+
"special": false
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"id": 57534,
|
| 161 |
+
"content": "<dot>",
|
| 162 |
+
"single_word": false,
|
| 163 |
+
"lstrip": false,
|
| 164 |
+
"rstrip": false,
|
| 165 |
+
"normalized": true,
|
| 166 |
+
"special": false
|
| 167 |
}
|
| 168 |
],
|
| 169 |
"normalizer": {
|