currentfear commited on
Commit
2f4faca
·
verified ·
1 Parent(s): 19ad9ec

Training done

Browse files
Files changed (3) hide show
  1. added_tokens.json +8 -8
  2. tokenizer.json +10 -10
  3. tokenizer_config.json +8 -8
added_tokens.json CHANGED
@@ -1,16 +1,16 @@
1
  {
2
- "</s_firstSurname>": 57534,
3
  "</s_name>": 57532,
4
- "</s_number>": 57530,
5
- "</s_secondSurname>": 57528,
6
- "</s_type>": 57526,
7
  "<s_cord-v2>": 57535,
8
- "<s_firstSurname>": 57533,
9
  "<s_iitcdip>": 57523,
10
  "<s_name>": 57531,
11
- "<s_number>": 57529,
12
- "<s_secondSurname>": 57527,
13
  "<s_synthdog>": 57524,
14
- "<s_type>": 57525,
15
  "<sep/>": 57522
16
  }
 
1
  {
2
+ "</s_firstSurname>": 57528,
3
  "</s_name>": 57532,
4
+ "</s_number>": 57526,
5
+ "</s_secondSurname>": 57530,
6
+ "</s_type>": 57534,
7
  "<s_cord-v2>": 57535,
8
+ "<s_firstSurname>": 57527,
9
  "<s_iitcdip>": 57523,
10
  "<s_name>": 57531,
11
+ "<s_number>": 57525,
12
+ "<s_secondSurname>": 57529,
13
  "<s_synthdog>": 57524,
14
+ "<s_type>": 57533,
15
  "<sep/>": 57522
16
  }
tokenizer.json CHANGED
@@ -2,13 +2,13 @@
2
  "version": "1.0",
3
  "truncation": {
4
  "direction": "Right",
5
- "max_length": 768,
6
  "strategy": "LongestFirst",
7
  "stride": 0
8
  },
9
  "padding": {
10
  "strategy": {
11
- "Fixed": 768
12
  },
13
  "direction": "Right",
14
  "pad_to_multiple_of": null,
@@ -91,7 +91,7 @@
91
  },
92
  {
93
  "id": 57525,
94
- "content": "<s_type>",
95
  "single_word": false,
96
  "lstrip": false,
97
  "rstrip": false,
@@ -100,7 +100,7 @@
100
  },
101
  {
102
  "id": 57526,
103
- "content": "</s_type>",
104
  "single_word": false,
105
  "lstrip": false,
106
  "rstrip": false,
@@ -109,7 +109,7 @@
109
  },
110
  {
111
  "id": 57527,
112
- "content": "<s_secondSurname>",
113
  "single_word": false,
114
  "lstrip": false,
115
  "rstrip": false,
@@ -118,7 +118,7 @@
118
  },
119
  {
120
  "id": 57528,
121
- "content": "</s_secondSurname>",
122
  "single_word": false,
123
  "lstrip": false,
124
  "rstrip": false,
@@ -127,7 +127,7 @@
127
  },
128
  {
129
  "id": 57529,
130
- "content": "<s_number>",
131
  "single_word": false,
132
  "lstrip": false,
133
  "rstrip": false,
@@ -136,7 +136,7 @@
136
  },
137
  {
138
  "id": 57530,
139
- "content": "</s_number>",
140
  "single_word": false,
141
  "lstrip": false,
142
  "rstrip": false,
@@ -163,7 +163,7 @@
163
  },
164
  {
165
  "id": 57533,
166
- "content": "<s_firstSurname>",
167
  "single_word": false,
168
  "lstrip": false,
169
  "rstrip": false,
@@ -172,7 +172,7 @@
172
  },
173
  {
174
  "id": 57534,
175
- "content": "</s_firstSurname>",
176
  "single_word": false,
177
  "lstrip": false,
178
  "rstrip": false,
 
2
  "version": "1.0",
3
  "truncation": {
4
  "direction": "Right",
5
+ "max_length": 512,
6
  "strategy": "LongestFirst",
7
  "stride": 0
8
  },
9
  "padding": {
10
  "strategy": {
11
+ "Fixed": 512
12
  },
13
  "direction": "Right",
14
  "pad_to_multiple_of": null,
 
91
  },
92
  {
93
  "id": 57525,
94
+ "content": "<s_number>",
95
  "single_word": false,
96
  "lstrip": false,
97
  "rstrip": false,
 
100
  },
101
  {
102
  "id": 57526,
103
+ "content": "</s_number>",
104
  "single_word": false,
105
  "lstrip": false,
106
  "rstrip": false,
 
109
  },
110
  {
111
  "id": 57527,
112
+ "content": "<s_firstSurname>",
113
  "single_word": false,
114
  "lstrip": false,
115
  "rstrip": false,
 
118
  },
119
  {
120
  "id": 57528,
121
+ "content": "</s_firstSurname>",
122
  "single_word": false,
123
  "lstrip": false,
124
  "rstrip": false,
 
127
  },
128
  {
129
  "id": 57529,
130
+ "content": "<s_secondSurname>",
131
  "single_word": false,
132
  "lstrip": false,
133
  "rstrip": false,
 
136
  },
137
  {
138
  "id": 57530,
139
+ "content": "</s_secondSurname>",
140
  "single_word": false,
141
  "lstrip": false,
142
  "rstrip": false,
 
163
  },
164
  {
165
  "id": 57533,
166
+ "content": "<s_type>",
167
  "single_word": false,
168
  "lstrip": false,
169
  "rstrip": false,
 
172
  },
173
  {
174
  "id": 57534,
175
+ "content": "</s_type>",
176
  "single_word": false,
177
  "lstrip": false,
178
  "rstrip": false,
tokenizer_config.json CHANGED
@@ -65,7 +65,7 @@
65
  "special": true
66
  },
67
  "57525": {
68
- "content": "<s_type>",
69
  "lstrip": false,
70
  "normalized": true,
71
  "rstrip": false,
@@ -73,7 +73,7 @@
73
  "special": false
74
  },
75
  "57526": {
76
- "content": "</s_type>",
77
  "lstrip": false,
78
  "normalized": true,
79
  "rstrip": false,
@@ -81,7 +81,7 @@
81
  "special": false
82
  },
83
  "57527": {
84
- "content": "<s_secondSurname>",
85
  "lstrip": false,
86
  "normalized": true,
87
  "rstrip": false,
@@ -89,7 +89,7 @@
89
  "special": false
90
  },
91
  "57528": {
92
- "content": "</s_secondSurname>",
93
  "lstrip": false,
94
  "normalized": true,
95
  "rstrip": false,
@@ -97,7 +97,7 @@
97
  "special": false
98
  },
99
  "57529": {
100
- "content": "<s_number>",
101
  "lstrip": false,
102
  "normalized": true,
103
  "rstrip": false,
@@ -105,7 +105,7 @@
105
  "special": false
106
  },
107
  "57530": {
108
- "content": "</s_number>",
109
  "lstrip": false,
110
  "normalized": true,
111
  "rstrip": false,
@@ -129,7 +129,7 @@
129
  "special": false
130
  },
131
  "57533": {
132
- "content": "<s_firstSurname>",
133
  "lstrip": false,
134
  "normalized": true,
135
  "rstrip": false,
@@ -137,7 +137,7 @@
137
  "special": false
138
  },
139
  "57534": {
140
- "content": "</s_firstSurname>",
141
  "lstrip": false,
142
  "normalized": true,
143
  "rstrip": false,
 
65
  "special": true
66
  },
67
  "57525": {
68
+ "content": "<s_number>",
69
  "lstrip": false,
70
  "normalized": true,
71
  "rstrip": false,
 
73
  "special": false
74
  },
75
  "57526": {
76
+ "content": "</s_number>",
77
  "lstrip": false,
78
  "normalized": true,
79
  "rstrip": false,
 
81
  "special": false
82
  },
83
  "57527": {
84
+ "content": "<s_firstSurname>",
85
  "lstrip": false,
86
  "normalized": true,
87
  "rstrip": false,
 
89
  "special": false
90
  },
91
  "57528": {
92
+ "content": "</s_firstSurname>",
93
  "lstrip": false,
94
  "normalized": true,
95
  "rstrip": false,
 
97
  "special": false
98
  },
99
  "57529": {
100
+ "content": "<s_secondSurname>",
101
  "lstrip": false,
102
  "normalized": true,
103
  "rstrip": false,
 
105
  "special": false
106
  },
107
  "57530": {
108
+ "content": "</s_secondSurname>",
109
  "lstrip": false,
110
  "normalized": true,
111
  "rstrip": false,
 
129
  "special": false
130
  },
131
  "57533": {
132
+ "content": "<s_type>",
133
  "lstrip": false,
134
  "normalized": true,
135
  "rstrip": false,
 
137
  "special": false
138
  },
139
  "57534": {
140
+ "content": "</s_type>",
141
  "lstrip": false,
142
  "normalized": true,
143
  "rstrip": false,