Training done
Browse files- added_tokens.json +3 -1
- tokenizer.json +18 -0
- tokenizer_config.json +16 -0
added_tokens.json
CHANGED
|
@@ -1,10 +1,12 @@
|
|
| 1 |
{
|
|
|
|
| 2 |
"</s_firstSurname>": 57528,
|
| 3 |
"</s_name>": 57532,
|
| 4 |
"</s_number>": 57526,
|
| 5 |
"</s_secondSurname>": 57530,
|
| 6 |
"</s_type>": 57534,
|
| 7 |
-
"<
|
|
|
|
| 8 |
"<s_firstSurname>": 57527,
|
| 9 |
"<s_iitcdip>": 57523,
|
| 10 |
"<s_name>": 57531,
|
|
|
|
| 1 |
{
|
| 2 |
+
"</s_cedula>": 57536,
|
| 3 |
"</s_firstSurname>": 57528,
|
| 4 |
"</s_name>": 57532,
|
| 5 |
"</s_number>": 57526,
|
| 6 |
"</s_secondSurname>": 57530,
|
| 7 |
"</s_type>": 57534,
|
| 8 |
+
"<s_cedula>": 57535,
|
| 9 |
+
"<s_cord-v2>": 57537,
|
| 10 |
"<s_firstSurname>": 57527,
|
| 11 |
"<s_iitcdip>": 57523,
|
| 12 |
"<s_name>": 57531,
|
tokenizer.json
CHANGED
|
@@ -181,6 +181,24 @@
|
|
| 181 |
},
|
| 182 |
{
|
| 183 |
"id": 57535,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
"content": "<s_cord-v2>",
|
| 185 |
"single_word": false,
|
| 186 |
"lstrip": false,
|
|
|
|
| 181 |
},
|
| 182 |
{
|
| 183 |
"id": 57535,
|
| 184 |
+
"content": "<s_cedula>",
|
| 185 |
+
"single_word": false,
|
| 186 |
+
"lstrip": false,
|
| 187 |
+
"rstrip": false,
|
| 188 |
+
"normalized": true,
|
| 189 |
+
"special": false
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"id": 57536,
|
| 193 |
+
"content": "</s_cedula>",
|
| 194 |
+
"single_word": false,
|
| 195 |
+
"lstrip": false,
|
| 196 |
+
"rstrip": false,
|
| 197 |
+
"normalized": true,
|
| 198 |
+
"special": false
|
| 199 |
+
},
|
| 200 |
+
{
|
| 201 |
+
"id": 57537,
|
| 202 |
"content": "<s_cord-v2>",
|
| 203 |
"single_word": false,
|
| 204 |
"lstrip": false,
|
tokenizer_config.json
CHANGED
|
@@ -145,6 +145,22 @@
|
|
| 145 |
"special": false
|
| 146 |
},
|
| 147 |
"57535": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
"content": "<s_cord-v2>",
|
| 149 |
"lstrip": false,
|
| 150 |
"normalized": true,
|
|
|
|
| 145 |
"special": false
|
| 146 |
},
|
| 147 |
"57535": {
|
| 148 |
+
"content": "<s_cedula>",
|
| 149 |
+
"lstrip": false,
|
| 150 |
+
"normalized": true,
|
| 151 |
+
"rstrip": false,
|
| 152 |
+
"single_word": false,
|
| 153 |
+
"special": false
|
| 154 |
+
},
|
| 155 |
+
"57536": {
|
| 156 |
+
"content": "</s_cedula>",
|
| 157 |
+
"lstrip": false,
|
| 158 |
+
"normalized": true,
|
| 159 |
+
"rstrip": false,
|
| 160 |
+
"single_word": false,
|
| 161 |
+
"special": false
|
| 162 |
+
},
|
| 163 |
+
"57537": {
|
| 164 |
"content": "<s_cord-v2>",
|
| 165 |
"lstrip": false,
|
| 166 |
"normalized": true,
|