currentfear commited on
Commit
2197264
·
verified ·
1 Parent(s): c54a8db

Training done

Browse files
Files changed (3) hide show
  1. added_tokens.json +3 -1
  2. tokenizer.json +18 -0
  3. tokenizer_config.json +16 -0
added_tokens.json CHANGED
@@ -1,10 +1,12 @@
1
  {
 
2
  "</s_firstSurname>": 57528,
3
  "</s_name>": 57532,
4
  "</s_number>": 57526,
5
  "</s_secondSurname>": 57530,
6
  "</s_type>": 57534,
7
- "<s_cord-v2>": 57535,
 
8
  "<s_firstSurname>": 57527,
9
  "<s_iitcdip>": 57523,
10
  "<s_name>": 57531,
 
1
  {
2
+ "</s_cedula>": 57536,
3
  "</s_firstSurname>": 57528,
4
  "</s_name>": 57532,
5
  "</s_number>": 57526,
6
  "</s_secondSurname>": 57530,
7
  "</s_type>": 57534,
8
+ "<s_cedula>": 57535,
9
+ "<s_cord-v2>": 57537,
10
  "<s_firstSurname>": 57527,
11
  "<s_iitcdip>": 57523,
12
  "<s_name>": 57531,
tokenizer.json CHANGED
@@ -181,6 +181,24 @@
181
  },
182
  {
183
  "id": 57535,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
184
  "content": "<s_cord-v2>",
185
  "single_word": false,
186
  "lstrip": false,
 
181
  },
182
  {
183
  "id": 57535,
184
+ "content": "<s_cedula>",
185
+ "single_word": false,
186
+ "lstrip": false,
187
+ "rstrip": false,
188
+ "normalized": true,
189
+ "special": false
190
+ },
191
+ {
192
+ "id": 57536,
193
+ "content": "</s_cedula>",
194
+ "single_word": false,
195
+ "lstrip": false,
196
+ "rstrip": false,
197
+ "normalized": true,
198
+ "special": false
199
+ },
200
+ {
201
+ "id": 57537,
202
  "content": "<s_cord-v2>",
203
  "single_word": false,
204
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -145,6 +145,22 @@
145
  "special": false
146
  },
147
  "57535": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  "content": "<s_cord-v2>",
149
  "lstrip": false,
150
  "normalized": true,
 
145
  "special": false
146
  },
147
  "57535": {
148
+ "content": "<s_cedula>",
149
+ "lstrip": false,
150
+ "normalized": true,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": false
154
+ },
155
+ "57536": {
156
+ "content": "</s_cedula>",
157
+ "lstrip": false,
158
+ "normalized": true,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": false
162
+ },
163
+ "57537": {
164
  "content": "<s_cord-v2>",
165
  "lstrip": false,
166
  "normalized": true,