codebyzeb commited on
Commit
b3b5b2d
·
verified ·
1 Parent(s): ad3d889

Delete fw57M_Entropy_thresholdM_500

Browse files
fw57M_Entropy_thresholdM_500/merges.txt DELETED
@@ -1,243 +0,0 @@
1
- #version: 0.2
2
- n g
3
- g e
4
- h e
5
- u r
6
- e n
7
- v e
8
- e r
9
- n t
10
- e t
11
- g h
12
- i g
13
- s s
14
- m e
15
- a r
16
- i ng
17
- o n
18
- r d
19
- n d
20
- d y
21
- l e
22
- d e
23
- l l
24
- l y
25
- o g
26
- i n
27
- h er
28
- d en
29
- t e
30
- c l
31
- l i
32
- r a
33
- c h
34
- o u
35
- e m
36
- k e
37
- i o
38
- t h
39
- a g
40
- c t
41
- a t
42
- a n
43
- r c
44
- e s
45
- s e
46
- o r
47
- u t
48
- g n
49
- a l
50
- e a
51
- e d
52
- o w
53
- u l
54
- l t
55
- t he
56
- u s
57
- p l
58
- u n
59
- c e
60
- en t
61
- t ur
62
- n e
63
- e l
64
- e c
65
- h t
66
- i l
67
- r e
68
- l d
69
- t u
70
- r g
71
- r t
72
- r o
73
- u d
74
- e ss
75
- a rd
76
- r i
77
- a d
78
- r v
79
- s t
80
- a m
81
- o m
82
- s h
83
- t i
84
- t r
85
- p le
86
- i a
87
- a p
88
- ec t
89
- q u
90
- u se
91
- t a
92
- a y
93
- g a
94
- ve r
95
- a nd
96
- i s
97
- i c
98
- h o
99
- u e
100
- io n
101
- i d
102
- l f
103
- b l
104
- o y
105
- h a
106
- n ce
107
- m a
108
- i v
109
- i t
110
- u g
111
- m p
112
- m b
113
- h i
114
- b ut
115
- n a
116
- a s
117
- u p
118
- c u
119
- u a
120
- t o
121
- ul t
122
- p h
123
- c k
124
- t em
125
- ra t
126
- a c
127
- p o
128
- w e
129
- o d
130
- n c
131
- r n
132
- v i
133
- s k
134
- e ar
135
- d i
136
- d o
137
- o p
138
- u c
139
- l a
140
- k a
141
- c a
142
- u i
143
- b er
144
- i th
145
- r k
146
- p t
147
- n i
148
- b e
149
- s u
150
- e st
151
- t her
152
- i p
153
- c i
154
- i sh
155
- a ge
156
- s i
157
- o l
158
- a se
159
- u gh
160
- t y
161
- r m
162
- i ne
163
- i r
164
- l o
165
- u nd
166
- a dy
167
- p a
168
- m en
169
- l ly
170
- gh t
171
- a b
172
- f e
173
- i b
174
- o rk
175
- o o
176
- e e
177
- l it
178
- t al
179
- rc h
180
- n o
181
- l u
182
- e y
183
- c ia
184
- i ty
185
- l tur
186
- a i
187
- a v
188
- n al
189
- ur e
190
- m al
191
- u m
192
- g u
193
- t er
194
- i m
195
- a ti
196
- s o
197
- l ar
198
- l em
199
- c er
200
- ti on
201
- r y
202
- m at
203
- r ce
204
- n s
205
- a te
206
- me nt
207
- o s
208
- e ra
209
- t or
210
- b le
211
- e k
212
- n k
213
- m i
214
- w er
215
- e ther
216
- e en
217
- v a
218
- o us
219
- d u
220
- h in
221
- t io
222
- i al
223
- o re
224
- e as
225
- b r
226
- d er
227
- a in
228
- o ut
229
- f i
230
- d ing
231
- r s
232
- a tion
233
- i at
234
- a nt
235
- o t
236
- a re
237
- l op
238
- t p
239
- s tem
240
- o od
241
- u en
242
- a rch
243
- o ol
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fw57M_Entropy_thresholdM_500/special_tokens_map.json DELETED
@@ -1,5 +0,0 @@
1
- {
2
- "bos_token": "<|endoftext|>",
3
- "eos_token": "<|endoftext|>",
4
- "pad_token": "<|padding|>"
5
- }
 
 
 
 
 
 
fw57M_Entropy_thresholdM_500/stats.csv DELETED
@@ -1,319 +0,0 @@
1
- num_moves,vocab_size,unique_segments,threshold
2
- 0,258,0,5.133828295811327e-08
3
- 1000,258,0,9.790553576749517e-07
4
- 2000,258,1,1.4309060816231067e-06
5
- 3000,258,2,1.8229197848995682e-06
6
- 4000,259,4,2.1990540517435875e-06
7
- 5000,259,8,2.5248320980608696e-06
8
- 6000,259,10,2.8365827802190324e-06
9
- 7000,259,12,3.127169520666939e-06
10
- 8000,259,15,3.4406789382046554e-06
11
- 9000,259,16,3.7212562347122002e-06
12
- 10000,259,18,4.012302269984502e-06
13
- 11000,259,19,4.296648512536194e-06
14
- 12000,259,21,4.568334588839207e-06
15
- 13000,260,26,4.853022801398765e-06
16
- 14000,260,30,5.109339326736517e-06
17
- 15000,260,38,5.373932253860403e-06
18
- 16000,260,40,5.649672857543919e-06
19
- 17000,260,42,5.914777375437552e-06
20
- 18000,261,46,6.172143912408501e-06
21
- 19000,261,48,6.424897492252057e-06
22
- 20000,262,49,6.700956419081194e-06
23
- 21000,262,53,6.9617381086573005e-06
24
- 22000,262,56,7.22233698979835e-06
25
- 23000,263,58,7.478172392438864e-06
26
- 24000,263,64,7.740409273537807e-06
27
- 25000,263,66,8.001956302905455e-06
28
- 26000,263,70,8.264463758678176e-06
29
- 27000,264,76,8.528708349331282e-06
30
- 28000,264,79,8.773720765020698e-06
31
- 29000,264,81,9.04225635167677e-06
32
- 30000,265,85,9.290170055464841e-06
33
- 31000,266,91,9.528370355837978e-06
34
- 32000,266,92,9.79736796580255e-06
35
- 33000,267,95,1.0070482858282048e-05
36
- 34000,267,98,1.0339319487684406e-05
37
- 35000,267,100,1.0614803613862023e-05
38
- 36000,267,104,1.0890100384131074e-05
39
- 37000,268,106,1.1162222108396236e-05
40
- 38000,268,110,1.1431737220846117e-05
41
- 39000,268,110,1.1706975783454254e-05
42
- 40000,269,115,1.1987172001681756e-05
43
- 41000,270,121,1.224808602273697e-05
44
- 42000,271,123,1.2501597666414455e-05
45
- 43000,271,125,1.2784775208274368e-05
46
- 44000,271,127,1.307702677877387e-05
47
- 45000,273,128,1.3359001059143338e-05
48
- 46000,274,132,1.3647608284372836e-05
49
- 47000,275,133,1.3921197023591958e-05
50
- 48000,275,135,1.4208838365448173e-05
51
- 49000,275,136,1.4507069863611832e-05
52
- 50000,276,141,1.4814748283242807e-05
53
- 51000,276,143,1.5173626707110088e-05
54
- 52000,277,146,1.5574130884488113e-05
55
- 53000,278,148,1.6095349565148354e-05
56
- 54000,278,149,1.6789324945420958e-05
57
- 55000,279,152,1.7530710465507582e-05
58
- 56000,279,155,1.807898297556676e-05
59
- 57000,281,157,1.855349910329096e-05
60
- 58000,282,162,1.89148177014431e-05
61
- 59000,285,165,1.9255410734331235e-05
62
- 60000,285,169,1.9606726709753275e-05
63
- 61000,285,171,1.993944897549227e-05
64
- 62000,286,175,2.026474612648599e-05
65
- 63000,286,177,2.057795245491434e-05
66
- 64000,286,183,2.0897583453916013e-05
67
- 65000,288,185,2.1221829229034483e-05
68
- 66000,290,188,2.1535339328693226e-05
69
- 67000,293,194,2.1876883693039417e-05
70
- 68000,293,195,2.2192627511685714e-05
71
- 69000,295,199,2.2512333089252934e-05
72
- 70000,297,200,2.281865272379946e-05
73
- 71000,299,204,2.3127340682549402e-05
74
- 72000,299,208,2.3458222131012008e-05
75
- 73000,299,214,2.3790689738234505e-05
76
- 74000,301,215,2.412626963632647e-05
77
- 75000,301,216,2.4468832634738646e-05
78
- 76000,303,217,2.481048068148084e-05
79
- 77000,305,221,2.5139239369309507e-05
80
- 78000,305,224,2.5478828320046887e-05
81
- 79000,305,228,2.5810599254327826e-05
82
- 80000,305,233,2.6150766643695533e-05
83
- 81000,305,237,2.6495927158975974e-05
84
- 82000,306,239,2.6845029424293898e-05
85
- 83000,306,241,2.7201735065318644e-05
86
- 84000,306,245,2.755965215328615e-05
87
- 85000,306,249,2.7887075702892616e-05
88
- 86000,309,254,2.823129943863023e-05
89
- 87000,309,255,2.858073821698781e-05
90
- 88000,309,256,2.8935739464941435e-05
91
- 89000,310,260,2.9296392312971875e-05
92
- 90000,310,264,2.9659753636224195e-05
93
- 91000,310,270,3.002668745466508e-05
94
- 92000,313,274,3.0397197406273335e-05
95
- 93000,313,280,3.074545384151861e-05
96
- 94000,313,282,3.1101779313758016e-05
97
- 95000,313,287,3.149575786665082e-05
98
- 96000,314,289,3.189174094586633e-05
99
- 97000,315,291,3.22468149533961e-05
100
- 98000,318,296,3.26372537529096e-05
101
- 99000,320,301,3.3024651202140376e-05
102
- 100000,320,304,3.339877730468288e-05
103
- 101000,320,314,3.377669054316357e-05
104
- 102000,320,321,3.417441257624887e-05
105
- 103000,320,325,3.456857302808203e-05
106
- 104000,320,331,3.496912540867925e-05
107
- 105000,322,340,3.5385990486247465e-05
108
- 106000,324,345,3.575959999579936e-05
109
- 107000,326,350,3.61450802301988e-05
110
- 108000,327,352,3.653268504422158e-05
111
- 109000,328,357,3.6942012229701504e-05
112
- 110000,331,365,3.730935713974759e-05
113
- 111000,332,370,3.769703835132532e-05
114
- 112000,333,377,3.8120309909572825e-05
115
- 113000,335,381,3.853967791656032e-05
116
- 114000,335,384,3.8975511415628716e-05
117
- 115000,336,391,3.93725567846559e-05
118
- 116000,337,395,3.9807622670196e-05
119
- 117000,337,403,4.022541907033883e-05
120
- 118000,339,405,4.067840927746147e-05
121
- 119000,339,410,4.113887916901149e-05
122
- 120000,340,416,4.1620303818490356e-05
123
- 121000,341,420,4.212183193885721e-05
124
- 122000,341,426,4.26527694799006e-05
125
- 123000,342,430,4.322654058341868e-05
126
- 124000,342,431,4.3803440348710865e-05
127
- 125000,343,436,4.446713501238264e-05
128
- 126000,343,442,4.5173615944804624e-05
129
- 127000,343,447,4.5849425077904016e-05
130
- 128000,345,453,4.65281555079855e-05
131
- 129000,346,460,4.7097088099690154e-05
132
- 130000,346,466,4.766680285683833e-05
133
- 131000,346,470,4.817744775209576e-05
134
- 132000,347,476,4.870327393291518e-05
135
- 133000,348,478,4.920174251310527e-05
136
- 134000,349,487,4.968308348907158e-05
137
- 135000,353,491,5.017643343308009e-05
138
- 136000,354,498,5.0663868023548275e-05
139
- 137000,354,504,5.1141963922418654e-05
140
- 138000,354,510,5.164055619388819e-05
141
- 139000,355,517,5.2116614824626595e-05
142
- 140000,355,522,5.258953387965448e-05
143
- 141000,355,527,5.307243554852903e-05
144
- 142000,355,531,5.356439214665443e-05
145
- 143000,355,537,5.403726390795782e-05
146
- 144000,355,545,5.4523996368516237e-05
147
- 145000,355,551,5.499910548678599e-05
148
- 146000,355,562,5.549181150854565e-05
149
- 147000,355,567,5.5992106354096904e-05
150
- 148000,355,571,5.6516106269555166e-05
151
- 149000,355,582,5.702984344679862e-05
152
- 150000,356,586,5.753296136390418e-05
153
- 151000,356,595,5.804073589388281e-05
154
- 152000,357,603,5.855420749867335e-05
155
- 153000,358,610,5.9088684793096036e-05
156
- 154000,359,618,5.9630576288327575e-05
157
- 155000,359,627,6.0167138144606724e-05
158
- 156000,362,634,6.067429785616696e-05
159
- 157000,363,640,6.120924081187695e-05
160
- 158000,364,647,6.172269058879465e-05
161
- 159000,364,656,6.227569247130305e-05
162
- 160000,366,662,6.282198592089117e-05
163
- 161000,367,669,6.337612285278738e-05
164
- 162000,367,674,6.394451338564977e-05
165
- 163000,367,678,6.452309025917202e-05
166
- 164000,367,684,6.510177627205849e-05
167
- 165000,368,689,6.569121615029871e-05
168
- 166000,369,701,6.630591815337539e-05
169
- 167000,369,713,6.693345494568348e-05
170
- 168000,371,720,6.759059760952368e-05
171
- 169000,374,729,6.822018622187898e-05
172
- 170000,375,738,6.889564247103408e-05
173
- 171000,375,742,6.960536848055199e-05
174
- 172000,376,753,7.03365876688622e-05
175
- 173000,378,760,7.111056038411334e-05
176
- 174000,379,769,7.18764858902432e-05
177
- 175000,379,778,7.268117769854143e-05
178
- 176000,381,787,7.343306788243353e-05
179
- 177000,383,795,7.421991904266179e-05
180
- 178000,384,800,7.493997691199183e-05
181
- 179000,385,813,7.568250293843448e-05
182
- 180000,388,819,7.634618668816984e-05
183
- 181000,390,827,7.700482092332095e-05
184
- 182000,390,834,7.760989683447406e-05
185
- 183000,390,839,7.826130604371428e-05
186
- 184000,391,851,7.892258145147935e-05
187
- 185000,393,858,7.959445065353066e-05
188
- 186000,394,862,8.023567352211103e-05
189
- 187000,395,869,8.084762521320954e-05
190
- 188000,395,877,8.148838969646022e-05
191
- 189000,396,886,8.213626279029995e-05
192
- 190000,396,894,8.276363223558292e-05
193
- 191000,396,902,8.341415377799422e-05
194
- 192000,396,913,8.410123700741678e-05
195
- 193000,396,917,8.473030175082386e-05
196
- 194000,397,931,8.538847032468766e-05
197
- 195000,398,941,8.607128984294832e-05
198
- 196000,398,946,8.677331788931042e-05
199
- 197000,399,953,8.74913384905085e-05
200
- 198000,400,962,8.819071808829904e-05
201
- 199000,401,967,8.887527656042948e-05
202
- 200000,401,974,8.96106066647917e-05
203
- 201000,403,982,9.039176802616566e-05
204
- 202000,404,996,9.116170258494094e-05
205
- 203000,406,1007,9.191161370836198e-05
206
- 204000,406,1013,9.268507710658014e-05
207
- 205000,406,1021,9.348066669190302e-05
208
- 206000,406,1025,9.431746002519503e-05
209
- 207000,407,1036,9.513343684375286e-05
210
- 208000,408,1047,9.600699559086934e-05
211
- 209000,408,1055,9.687518468126655e-05
212
- 210000,408,1064,9.775336366146803e-05
213
- 211000,408,1077,9.860962745733559e-05
214
- 212000,410,1087,9.952823893399909e-05
215
- 213000,411,1092,0.00010041777568403631
216
- 214000,412,1106,0.00010130731971003115
217
- 215000,412,1113,0.0001021612697513774
218
- 216000,413,1117,0.00010298722190782428
219
- 217000,415,1126,0.0001038184855133295
220
- 218000,415,1132,0.00010466543608345091
221
- 219000,417,1139,0.00010548390855547041
222
- 220000,418,1142,0.0001063163872458972
223
- 221000,418,1148,0.0001070987491402775
224
- 222000,419,1154,0.00010796061542350799
225
- 223000,420,1166,0.00010877475142478943
226
- 224000,421,1172,0.00010960102372337133
227
- 225000,421,1179,0.00011046299187000841
228
- 226000,422,1188,0.00011130447092000395
229
- 227000,422,1205,0.00011216900747967884
230
- 228000,422,1213,0.00011303589417366311
231
- 229000,422,1218,0.00011391702719265595
232
- 230000,423,1229,0.00011477310908958316
233
- 231000,424,1239,0.00011569355410756543
234
- 232000,425,1248,0.00011659049050649628
235
- 233000,425,1253,0.00011754433216992766
236
- 234000,426,1260,0.0001184323919005692
237
- 235000,426,1275,0.00011939684918615967
238
- 236000,427,1285,0.00012038474233122543
239
- 237000,429,1295,0.0001214050134876743
240
- 238000,430,1305,0.00012243652599863708
241
- 239000,430,1313,0.0001234794472111389
242
- 240000,431,1323,0.00012448015331756324
243
- 241000,431,1336,0.00012551326653920114
244
- 242000,434,1339,0.00012652068107854575
245
- 243000,436,1352,0.00012746831635013223
246
- 244000,436,1361,0.00012846790195908397
247
- 245000,436,1371,0.00012941929162479937
248
- 246000,436,1378,0.000130387008539401
249
- 247000,437,1385,0.0001313605607720092
250
- 248000,437,1397,0.00013233150821179152
251
- 249000,440,1408,0.00013329619832802564
252
- 250000,440,1417,0.00013422507618088275
253
- 251000,440,1427,0.00013517335173673928
254
- 252000,442,1438,0.00013617899094242603
255
- 253000,442,1447,0.00013713783118873835
256
- 254000,442,1455,0.00013816852879244834
257
- 255000,442,1464,0.00013914502051193267
258
- 256000,446,1477,0.00014017309877090156
259
- 257000,446,1494,0.0001412533165421337
260
- 258000,447,1510,0.00014228984946385026
261
- 259000,447,1521,0.00014341094356495887
262
- 260000,448,1533,0.00014451243623625487
263
- 261000,449,1538,0.00014568472397513688
264
- 262000,451,1550,0.00014690117677673697
265
- 263000,451,1561,0.00014807340630795807
266
- 264000,451,1570,0.0001492309384047985
267
- 265000,451,1583,0.0001503805397078395
268
- 266000,453,1590,0.00015156104927882552
269
- 267000,453,1598,0.00015271393931470811
270
- 268000,453,1618,0.00015389203326776624
271
- 269000,454,1633,0.00015501159941777587
272
- 270000,454,1648,0.00015614868607372046
273
- 271000,456,1657,0.00015728520520497113
274
- 272000,457,1667,0.0001584065321367234
275
- 273000,458,1676,0.00015955821436364204
276
- 274000,460,1683,0.00016068721015471965
277
- 275000,461,1693,0.00016185276035685092
278
- 276000,464,1707,0.00016305287135764956
279
- 277000,465,1716,0.00016424903878942132
280
- 278000,466,1731,0.00016550265718251467
281
- 279000,466,1744,0.0001666881435085088
282
- 280000,466,1758,0.0001679726701695472
283
- 281000,467,1767,0.00016916971071623266
284
- 282000,467,1779,0.00017038031364791095
285
- 283000,470,1791,0.00017168864724226296
286
- 284000,470,1799,0.0001728986098896712
287
- 285000,471,1809,0.00017415231559425592
288
- 286000,472,1817,0.0001754020049702376
289
- 287000,474,1825,0.00017670051602181047
290
- 288000,475,1836,0.00017809536075219512
291
- 289000,475,1844,0.00017944462888408452
292
- 290000,477,1855,0.00018079427536576986
293
- 291000,477,1862,0.0001821514160837978
294
- 292000,477,1879,0.00018344901036471128
295
- 293000,478,1890,0.0001847625244408846
296
- 294000,478,1907,0.00018616612942423671
297
- 295000,479,1917,0.00018746851128526032
298
- 296000,481,1928,0.00018876550893764943
299
- 297000,482,1943,0.0001900622883113101
300
- 298000,484,1960,0.0001915221946546808
301
- 299000,486,1968,0.00019285913731437176
302
- 300000,486,1977,0.0001942789094755426
303
- 301000,486,1990,0.00019575172336772084
304
- 302000,487,2002,0.00019722759316209704
305
- 303000,489,2009,0.00019872430129908025
306
- 304000,492,2018,0.0002001468965318054
307
- 305000,493,2029,0.00020151579519733787
308
- 306000,493,2041,0.00020292762201279402
309
- 307000,493,2053,0.00020436220802366734
310
- 308000,493,2062,0.00020579525153152645
311
- 309000,495,2070,0.00020730339747387916
312
- 310000,496,2078,0.00020878612122032791
313
- 311000,496,2090,0.00021032213408034295
314
- 312000,496,2103,0.0002117837138939649
315
- 313000,497,2111,0.0002133245870936662
316
- 314000,498,2121,0.00021483484306372702
317
- 315000,498,2131,0.00021635045413859189
318
- 316000,499,2139,0.00021790519531350583
319
- 317000,501,2152,0.00021950664813630283
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fw57M_Entropy_thresholdM_500/tokenizer.json DELETED
@@ -1,1533 +0,0 @@
1
- {
2
- "version": "1.0",
3
- "truncation": null,
4
- "padding": null,
5
- "added_tokens": [
6
- {
7
- "id": 0,
8
- "content": "<|padding|>",
9
- "single_word": false,
10
- "lstrip": false,
11
- "rstrip": false,
12
- "normalized": false,
13
- "special": true
14
- },
15
- {
16
- "id": 1,
17
- "content": "<|endoftext|>",
18
- "single_word": false,
19
- "lstrip": false,
20
- "rstrip": false,
21
- "normalized": false,
22
- "special": true
23
- }
24
- ],
25
- "normalizer": {
26
- "type": "Sequence",
27
- "normalizers": [
28
- {
29
- "type": "NFD"
30
- }
31
- ]
32
- },
33
- "pre_tokenizer": {
34
- "type": "ByteLevel",
35
- "add_prefix_space": true,
36
- "trim_offsets": true,
37
- "use_regex": true
38
- },
39
- "post_processor": {
40
- "type": "ByteLevel",
41
- "add_prefix_space": true,
42
- "trim_offsets": true,
43
- "use_regex": true
44
- },
45
- "decoder": {
46
- "type": "ByteLevel",
47
- "add_prefix_space": true,
48
- "trim_offsets": true,
49
- "use_regex": true
50
- },
51
- "model": {
52
- "type": "BPE",
53
- "dropout": null,
54
- "unk_token": null,
55
- "continuing_subword_prefix": null,
56
- "end_of_word_suffix": null,
57
- "fuse_unk": false,
58
- "byte_fallback": false,
59
- "ignore_merges": false,
60
- "vocab": {
61
- "<|padding|>": 0,
62
- "<|endoftext|>": 1,
63
- "!": 2,
64
- "\"": 3,
65
- "#": 4,
66
- "$": 5,
67
- "%": 6,
68
- "&": 7,
69
- "'": 8,
70
- "(": 9,
71
- ")": 10,
72
- "*": 11,
73
- "+": 12,
74
- ",": 13,
75
- "-": 14,
76
- ".": 15,
77
- "/": 16,
78
- "0": 17,
79
- "1": 18,
80
- "2": 19,
81
- "3": 20,
82
- "4": 21,
83
- "5": 22,
84
- "6": 23,
85
- "7": 24,
86
- "8": 25,
87
- "9": 26,
88
- ":": 27,
89
- ";": 28,
90
- "<": 29,
91
- "=": 30,
92
- ">": 31,
93
- "?": 32,
94
- "@": 33,
95
- "A": 34,
96
- "B": 35,
97
- "C": 36,
98
- "D": 37,
99
- "E": 38,
100
- "F": 39,
101
- "G": 40,
102
- "H": 41,
103
- "I": 42,
104
- "J": 43,
105
- "K": 44,
106
- "L": 45,
107
- "M": 46,
108
- "N": 47,
109
- "O": 48,
110
- "P": 49,
111
- "Q": 50,
112
- "R": 51,
113
- "S": 52,
114
- "T": 53,
115
- "U": 54,
116
- "V": 55,
117
- "W": 56,
118
- "X": 57,
119
- "Y": 58,
120
- "Z": 59,
121
- "[": 60,
122
- "\\": 61,
123
- "]": 62,
124
- "^": 63,
125
- "_": 64,
126
- "`": 65,
127
- "a": 66,
128
- "b": 67,
129
- "c": 68,
130
- "d": 69,
131
- "e": 70,
132
- "f": 71,
133
- "g": 72,
134
- "h": 73,
135
- "i": 74,
136
- "j": 75,
137
- "k": 76,
138
- "l": 77,
139
- "m": 78,
140
- "n": 79,
141
- "o": 80,
142
- "p": 81,
143
- "q": 82,
144
- "r": 83,
145
- "s": 84,
146
- "t": 85,
147
- "u": 86,
148
- "v": 87,
149
- "w": 88,
150
- "x": 89,
151
- "y": 90,
152
- "z": 91,
153
- "{": 92,
154
- "|": 93,
155
- "}": 94,
156
- "~": 95,
157
- "¡": 96,
158
- "¢": 97,
159
- "£": 98,
160
- "¤": 99,
161
- "¥": 100,
162
- "¦": 101,
163
- "§": 102,
164
- "¨": 103,
165
- "©": 104,
166
- "ª": 105,
167
- "«": 106,
168
- "¬": 107,
169
- "®": 108,
170
- "¯": 109,
171
- "°": 110,
172
- "±": 111,
173
- "²": 112,
174
- "³": 113,
175
- "´": 114,
176
- "µ": 115,
177
- "¶": 116,
178
- "·": 117,
179
- "¸": 118,
180
- "¹": 119,
181
- "º": 120,
182
- "»": 121,
183
- "¼": 122,
184
- "½": 123,
185
- "¾": 124,
186
- "¿": 125,
187
- "À": 126,
188
- "Á": 127,
189
- "Â": 128,
190
- "Ã": 129,
191
- "Ä": 130,
192
- "Å": 131,
193
- "Æ": 132,
194
- "Ç": 133,
195
- "È": 134,
196
- "É": 135,
197
- "Ê": 136,
198
- "Ë": 137,
199
- "Ì": 138,
200
- "Í": 139,
201
- "Î": 140,
202
- "Ï": 141,
203
- "Ð": 142,
204
- "Ñ": 143,
205
- "Ò": 144,
206
- "Ó": 145,
207
- "Ô": 146,
208
- "Õ": 147,
209
- "Ö": 148,
210
- "×": 149,
211
- "Ø": 150,
212
- "Ù": 151,
213
- "Ú": 152,
214
- "Û": 153,
215
- "Ü": 154,
216
- "Ý": 155,
217
- "Þ": 156,
218
- "ß": 157,
219
- "à": 158,
220
- "á": 159,
221
- "â": 160,
222
- "ã": 161,
223
- "ä": 162,
224
- "å": 163,
225
- "æ": 164,
226
- "ç": 165,
227
- "è": 166,
228
- "é": 167,
229
- "ê": 168,
230
- "ë": 169,
231
- "ì": 170,
232
- "í": 171,
233
- "î": 172,
234
- "ï": 173,
235
- "ð": 174,
236
- "ñ": 175,
237
- "ò": 176,
238
- "ó": 177,
239
- "ô": 178,
240
- "õ": 179,
241
- "ö": 180,
242
- "÷": 181,
243
- "ø": 182,
244
- "ù": 183,
245
- "ú": 184,
246
- "û": 185,
247
- "ü": 186,
248
- "ý": 187,
249
- "þ": 188,
250
- "ÿ": 189,
251
- "Ā": 190,
252
- "ā": 191,
253
- "Ă": 192,
254
- "ă": 193,
255
- "Ą": 194,
256
- "ą": 195,
257
- "Ć": 196,
258
- "ć": 197,
259
- "Ĉ": 198,
260
- "ĉ": 199,
261
- "Ċ": 200,
262
- "ċ": 201,
263
- "Č": 202,
264
- "č": 203,
265
- "Ď": 204,
266
- "ď": 205,
267
- "Đ": 206,
268
- "đ": 207,
269
- "Ē": 208,
270
- "ē": 209,
271
- "Ĕ": 210,
272
- "ĕ": 211,
273
- "Ė": 212,
274
- "ė": 213,
275
- "Ę": 214,
276
- "ę": 215,
277
- "Ě": 216,
278
- "ě": 217,
279
- "Ĝ": 218,
280
- "ĝ": 219,
281
- "Ğ": 220,
282
- "ğ": 221,
283
- "Ġ": 222,
284
- "ġ": 223,
285
- "Ģ": 224,
286
- "ģ": 225,
287
- "Ĥ": 226,
288
- "ĥ": 227,
289
- "Ħ": 228,
290
- "ħ": 229,
291
- "Ĩ": 230,
292
- "ĩ": 231,
293
- "Ī": 232,
294
- "ī": 233,
295
- "Ĭ": 234,
296
- "ĭ": 235,
297
- "Į": 236,
298
- "į": 237,
299
- "İ": 238,
300
- "ı": 239,
301
- "IJ": 240,
302
- "ij": 241,
303
- "Ĵ": 242,
304
- "ĵ": 243,
305
- "Ķ": 244,
306
- "ķ": 245,
307
- "ĸ": 246,
308
- "Ĺ": 247,
309
- "ĺ": 248,
310
- "Ļ": 249,
311
- "ļ": 250,
312
- "Ľ": 251,
313
- "ľ": 252,
314
- "Ŀ": 253,
315
- "ŀ": 254,
316
- "Ł": 255,
317
- "ł": 256,
318
- "Ń": 257,
319
- "ng": 258,
320
- "ge": 259,
321
- "he": 260,
322
- "ur": 261,
323
- "en": 262,
324
- "ve": 263,
325
- "er": 264,
326
- "nt": 265,
327
- "et": 266,
328
- "gh": 267,
329
- "ig": 268,
330
- "ss": 269,
331
- "me": 270,
332
- "ar": 271,
333
- "ing": 272,
334
- "on": 273,
335
- "rd": 274,
336
- "nd": 275,
337
- "dy": 276,
338
- "le": 277,
339
- "de": 278,
340
- "ll": 279,
341
- "ly": 280,
342
- "og": 281,
343
- "in": 282,
344
- "her": 283,
345
- "den": 284,
346
- "te": 285,
347
- "cl": 286,
348
- "li": 287,
349
- "ra": 288,
350
- "ch": 289,
351
- "ou": 290,
352
- "em": 291,
353
- "ke": 292,
354
- "io": 293,
355
- "th": 294,
356
- "ag": 295,
357
- "ct": 296,
358
- "at": 297,
359
- "an": 298,
360
- "rc": 299,
361
- "es": 300,
362
- "se": 301,
363
- "or": 302,
364
- "ut": 303,
365
- "gn": 304,
366
- "al": 305,
367
- "ea": 306,
368
- "ed": 307,
369
- "ow": 308,
370
- "ul": 309,
371
- "lt": 310,
372
- "the": 311,
373
- "us": 312,
374
- "pl": 313,
375
- "un": 314,
376
- "ce": 315,
377
- "ent": 316,
378
- "tur": 317,
379
- "ne": 318,
380
- "el": 319,
381
- "ec": 320,
382
- "ht": 321,
383
- "il": 322,
384
- "re": 323,
385
- "ld": 324,
386
- "tu": 325,
387
- "rg": 326,
388
- "rt": 327,
389
- "ro": 328,
390
- "ud": 329,
391
- "ess": 330,
392
- "ard": 331,
393
- "ri": 332,
394
- "ad": 333,
395
- "rv": 334,
396
- "st": 335,
397
- "am": 336,
398
- "om": 337,
399
- "sh": 338,
400
- "ti": 339,
401
- "tr": 340,
402
- "ple": 341,
403
- "ia": 342,
404
- "ap": 343,
405
- "ect": 344,
406
- "qu": 345,
407
- "use": 346,
408
- "ta": 347,
409
- "ay": 348,
410
- "ga": 349,
411
- "ver": 350,
412
- "and": 351,
413
- "is": 352,
414
- "ic": 353,
415
- "ho": 354,
416
- "ue": 355,
417
- "ion": 356,
418
- "id": 357,
419
- "lf": 358,
420
- "bl": 359,
421
- "oy": 360,
422
- "ha": 361,
423
- "nce": 362,
424
- "ma": 363,
425
- "iv": 364,
426
- "it": 365,
427
- "ug": 366,
428
- "mp": 367,
429
- "mb": 368,
430
- "hi": 369,
431
- "but": 370,
432
- "na": 371,
433
- "as": 372,
434
- "up": 373,
435
- "cu": 374,
436
- "ua": 375,
437
- "to": 376,
438
- "ult": 377,
439
- "ph": 378,
440
- "ck": 379,
441
- "tem": 380,
442
- "rat": 381,
443
- "ac": 382,
444
- "po": 383,
445
- "we": 384,
446
- "od": 385,
447
- "nc": 386,
448
- "rn": 387,
449
- "vi": 388,
450
- "sk": 389,
451
- "ear": 390,
452
- "di": 391,
453
- "do": 392,
454
- "op": 393,
455
- "uc": 394,
456
- "la": 395,
457
- "ka": 396,
458
- "ca": 397,
459
- "ui": 398,
460
- "ber": 399,
461
- "ith": 400,
462
- "rk": 401,
463
- "pt": 402,
464
- "ni": 403,
465
- "be": 404,
466
- "su": 405,
467
- "est": 406,
468
- "ther": 407,
469
- "ip": 408,
470
- "ci": 409,
471
- "ish": 410,
472
- "age": 411,
473
- "si": 412,
474
- "ol": 413,
475
- "ase": 414,
476
- "ugh": 415,
477
- "ty": 416,
478
- "rm": 417,
479
- "ine": 418,
480
- "ir": 419,
481
- "lo": 420,
482
- "und": 421,
483
- "ady": 422,
484
- "pa": 423,
485
- "men": 424,
486
- "lly": 425,
487
- "ght": 426,
488
- "ab": 427,
489
- "fe": 428,
490
- "ib": 429,
491
- "ork": 430,
492
- "oo": 431,
493
- "ee": 432,
494
- "lit": 433,
495
- "tal": 434,
496
- "rch": 435,
497
- "no": 436,
498
- "lu": 437,
499
- "ey": 438,
500
- "cia": 439,
501
- "ity": 440,
502
- "ltur": 441,
503
- "ai": 442,
504
- "av": 443,
505
- "nal": 444,
506
- "ure": 445,
507
- "mal": 446,
508
- "um": 447,
509
- "gu": 448,
510
- "ter": 449,
511
- "im": 450,
512
- "ati": 451,
513
- "so": 452,
514
- "lar": 453,
515
- "lem": 454,
516
- "cer": 455,
517
- "tion": 456,
518
- "ry": 457,
519
- "mat": 458,
520
- "rce": 459,
521
- "ns": 460,
522
- "ate": 461,
523
- "ment": 462,
524
- "os": 463,
525
- "era": 464,
526
- "tor": 465,
527
- "ble": 466,
528
- "ek": 467,
529
- "nk": 468,
530
- "mi": 469,
531
- "wer": 470,
532
- "ether": 471,
533
- "een": 472,
534
- "va": 473,
535
- "ous": 474,
536
- "du": 475,
537
- "hin": 476,
538
- "tio": 477,
539
- "ial": 478,
540
- "ore": 479,
541
- "eas": 480,
542
- "br": 481,
543
- "der": 482,
544
- "ain": 483,
545
- "out": 484,
546
- "fi": 485,
547
- "ding": 486,
548
- "rs": 487,
549
- "ation": 488,
550
- "iat": 489,
551
- "ant": 490,
552
- "ot": 491,
553
- "are": 492,
554
- "lop": 493,
555
- "tp": 494,
556
- "stem": 495,
557
- "ood": 496,
558
- "uen": 497,
559
- "arch": 498,
560
- "ool": 499
561
- },
562
- "merges": [
563
- [
564
- "n",
565
- "g"
566
- ],
567
- [
568
- "g",
569
- "e"
570
- ],
571
- [
572
- "h",
573
- "e"
574
- ],
575
- [
576
- "u",
577
- "r"
578
- ],
579
- [
580
- "e",
581
- "n"
582
- ],
583
- [
584
- "v",
585
- "e"
586
- ],
587
- [
588
- "e",
589
- "r"
590
- ],
591
- [
592
- "n",
593
- "t"
594
- ],
595
- [
596
- "e",
597
- "t"
598
- ],
599
- [
600
- "g",
601
- "h"
602
- ],
603
- [
604
- "i",
605
- "g"
606
- ],
607
- [
608
- "s",
609
- "s"
610
- ],
611
- [
612
- "m",
613
- "e"
614
- ],
615
- [
616
- "a",
617
- "r"
618
- ],
619
- [
620
- "i",
621
- "ng"
622
- ],
623
- [
624
- "o",
625
- "n"
626
- ],
627
- [
628
- "r",
629
- "d"
630
- ],
631
- [
632
- "n",
633
- "d"
634
- ],
635
- [
636
- "d",
637
- "y"
638
- ],
639
- [
640
- "l",
641
- "e"
642
- ],
643
- [
644
- "d",
645
- "e"
646
- ],
647
- [
648
- "l",
649
- "l"
650
- ],
651
- [
652
- "l",
653
- "y"
654
- ],
655
- [
656
- "o",
657
- "g"
658
- ],
659
- [
660
- "i",
661
- "n"
662
- ],
663
- [
664
- "h",
665
- "er"
666
- ],
667
- [
668
- "d",
669
- "en"
670
- ],
671
- [
672
- "t",
673
- "e"
674
- ],
675
- [
676
- "c",
677
- "l"
678
- ],
679
- [
680
- "l",
681
- "i"
682
- ],
683
- [
684
- "r",
685
- "a"
686
- ],
687
- [
688
- "c",
689
- "h"
690
- ],
691
- [
692
- "o",
693
- "u"
694
- ],
695
- [
696
- "e",
697
- "m"
698
- ],
699
- [
700
- "k",
701
- "e"
702
- ],
703
- [
704
- "i",
705
- "o"
706
- ],
707
- [
708
- "t",
709
- "h"
710
- ],
711
- [
712
- "a",
713
- "g"
714
- ],
715
- [
716
- "c",
717
- "t"
718
- ],
719
- [
720
- "a",
721
- "t"
722
- ],
723
- [
724
- "a",
725
- "n"
726
- ],
727
- [
728
- "r",
729
- "c"
730
- ],
731
- [
732
- "e",
733
- "s"
734
- ],
735
- [
736
- "s",
737
- "e"
738
- ],
739
- [
740
- "o",
741
- "r"
742
- ],
743
- [
744
- "u",
745
- "t"
746
- ],
747
- [
748
- "g",
749
- "n"
750
- ],
751
- [
752
- "a",
753
- "l"
754
- ],
755
- [
756
- "e",
757
- "a"
758
- ],
759
- [
760
- "e",
761
- "d"
762
- ],
763
- [
764
- "o",
765
- "w"
766
- ],
767
- [
768
- "u",
769
- "l"
770
- ],
771
- [
772
- "l",
773
- "t"
774
- ],
775
- [
776
- "t",
777
- "he"
778
- ],
779
- [
780
- "u",
781
- "s"
782
- ],
783
- [
784
- "p",
785
- "l"
786
- ],
787
- [
788
- "u",
789
- "n"
790
- ],
791
- [
792
- "c",
793
- "e"
794
- ],
795
- [
796
- "en",
797
- "t"
798
- ],
799
- [
800
- "t",
801
- "ur"
802
- ],
803
- [
804
- "n",
805
- "e"
806
- ],
807
- [
808
- "e",
809
- "l"
810
- ],
811
- [
812
- "e",
813
- "c"
814
- ],
815
- [
816
- "h",
817
- "t"
818
- ],
819
- [
820
- "i",
821
- "l"
822
- ],
823
- [
824
- "r",
825
- "e"
826
- ],
827
- [
828
- "l",
829
- "d"
830
- ],
831
- [
832
- "t",
833
- "u"
834
- ],
835
- [
836
- "r",
837
- "g"
838
- ],
839
- [
840
- "r",
841
- "t"
842
- ],
843
- [
844
- "r",
845
- "o"
846
- ],
847
- [
848
- "u",
849
- "d"
850
- ],
851
- [
852
- "e",
853
- "ss"
854
- ],
855
- [
856
- "a",
857
- "rd"
858
- ],
859
- [
860
- "r",
861
- "i"
862
- ],
863
- [
864
- "a",
865
- "d"
866
- ],
867
- [
868
- "r",
869
- "v"
870
- ],
871
- [
872
- "s",
873
- "t"
874
- ],
875
- [
876
- "a",
877
- "m"
878
- ],
879
- [
880
- "o",
881
- "m"
882
- ],
883
- [
884
- "s",
885
- "h"
886
- ],
887
- [
888
- "t",
889
- "i"
890
- ],
891
- [
892
- "t",
893
- "r"
894
- ],
895
- [
896
- "p",
897
- "le"
898
- ],
899
- [
900
- "i",
901
- "a"
902
- ],
903
- [
904
- "a",
905
- "p"
906
- ],
907
- [
908
- "ec",
909
- "t"
910
- ],
911
- [
912
- "q",
913
- "u"
914
- ],
915
- [
916
- "u",
917
- "se"
918
- ],
919
- [
920
- "t",
921
- "a"
922
- ],
923
- [
924
- "a",
925
- "y"
926
- ],
927
- [
928
- "g",
929
- "a"
930
- ],
931
- [
932
- "ve",
933
- "r"
934
- ],
935
- [
936
- "a",
937
- "nd"
938
- ],
939
- [
940
- "i",
941
- "s"
942
- ],
943
- [
944
- "i",
945
- "c"
946
- ],
947
- [
948
- "h",
949
- "o"
950
- ],
951
- [
952
- "u",
953
- "e"
954
- ],
955
- [
956
- "io",
957
- "n"
958
- ],
959
- [
960
- "i",
961
- "d"
962
- ],
963
- [
964
- "l",
965
- "f"
966
- ],
967
- [
968
- "b",
969
- "l"
970
- ],
971
- [
972
- "o",
973
- "y"
974
- ],
975
- [
976
- "h",
977
- "a"
978
- ],
979
- [
980
- "n",
981
- "ce"
982
- ],
983
- [
984
- "m",
985
- "a"
986
- ],
987
- [
988
- "i",
989
- "v"
990
- ],
991
- [
992
- "i",
993
- "t"
994
- ],
995
- [
996
- "u",
997
- "g"
998
- ],
999
- [
1000
- "m",
1001
- "p"
1002
- ],
1003
- [
1004
- "m",
1005
- "b"
1006
- ],
1007
- [
1008
- "h",
1009
- "i"
1010
- ],
1011
- [
1012
- "b",
1013
- "ut"
1014
- ],
1015
- [
1016
- "n",
1017
- "a"
1018
- ],
1019
- [
1020
- "a",
1021
- "s"
1022
- ],
1023
- [
1024
- "u",
1025
- "p"
1026
- ],
1027
- [
1028
- "c",
1029
- "u"
1030
- ],
1031
- [
1032
- "u",
1033
- "a"
1034
- ],
1035
- [
1036
- "t",
1037
- "o"
1038
- ],
1039
- [
1040
- "ul",
1041
- "t"
1042
- ],
1043
- [
1044
- "p",
1045
- "h"
1046
- ],
1047
- [
1048
- "c",
1049
- "k"
1050
- ],
1051
- [
1052
- "t",
1053
- "em"
1054
- ],
1055
- [
1056
- "ra",
1057
- "t"
1058
- ],
1059
- [
1060
- "a",
1061
- "c"
1062
- ],
1063
- [
1064
- "p",
1065
- "o"
1066
- ],
1067
- [
1068
- "w",
1069
- "e"
1070
- ],
1071
- [
1072
- "o",
1073
- "d"
1074
- ],
1075
- [
1076
- "n",
1077
- "c"
1078
- ],
1079
- [
1080
- "r",
1081
- "n"
1082
- ],
1083
- [
1084
- "v",
1085
- "i"
1086
- ],
1087
- [
1088
- "s",
1089
- "k"
1090
- ],
1091
- [
1092
- "e",
1093
- "ar"
1094
- ],
1095
- [
1096
- "d",
1097
- "i"
1098
- ],
1099
- [
1100
- "d",
1101
- "o"
1102
- ],
1103
- [
1104
- "o",
1105
- "p"
1106
- ],
1107
- [
1108
- "u",
1109
- "c"
1110
- ],
1111
- [
1112
- "l",
1113
- "a"
1114
- ],
1115
- [
1116
- "k",
1117
- "a"
1118
- ],
1119
- [
1120
- "c",
1121
- "a"
1122
- ],
1123
- [
1124
- "u",
1125
- "i"
1126
- ],
1127
- [
1128
- "b",
1129
- "er"
1130
- ],
1131
- [
1132
- "i",
1133
- "th"
1134
- ],
1135
- [
1136
- "r",
1137
- "k"
1138
- ],
1139
- [
1140
- "p",
1141
- "t"
1142
- ],
1143
- [
1144
- "n",
1145
- "i"
1146
- ],
1147
- [
1148
- "b",
1149
- "e"
1150
- ],
1151
- [
1152
- "s",
1153
- "u"
1154
- ],
1155
- [
1156
- "e",
1157
- "st"
1158
- ],
1159
- [
1160
- "t",
1161
- "her"
1162
- ],
1163
- [
1164
- "i",
1165
- "p"
1166
- ],
1167
- [
1168
- "c",
1169
- "i"
1170
- ],
1171
- [
1172
- "i",
1173
- "sh"
1174
- ],
1175
- [
1176
- "a",
1177
- "ge"
1178
- ],
1179
- [
1180
- "s",
1181
- "i"
1182
- ],
1183
- [
1184
- "o",
1185
- "l"
1186
- ],
1187
- [
1188
- "a",
1189
- "se"
1190
- ],
1191
- [
1192
- "u",
1193
- "gh"
1194
- ],
1195
- [
1196
- "t",
1197
- "y"
1198
- ],
1199
- [
1200
- "r",
1201
- "m"
1202
- ],
1203
- [
1204
- "i",
1205
- "ne"
1206
- ],
1207
- [
1208
- "i",
1209
- "r"
1210
- ],
1211
- [
1212
- "l",
1213
- "o"
1214
- ],
1215
- [
1216
- "u",
1217
- "nd"
1218
- ],
1219
- [
1220
- "a",
1221
- "dy"
1222
- ],
1223
- [
1224
- "p",
1225
- "a"
1226
- ],
1227
- [
1228
- "m",
1229
- "en"
1230
- ],
1231
- [
1232
- "l",
1233
- "ly"
1234
- ],
1235
- [
1236
- "gh",
1237
- "t"
1238
- ],
1239
- [
1240
- "a",
1241
- "b"
1242
- ],
1243
- [
1244
- "f",
1245
- "e"
1246
- ],
1247
- [
1248
- "i",
1249
- "b"
1250
- ],
1251
- [
1252
- "o",
1253
- "rk"
1254
- ],
1255
- [
1256
- "o",
1257
- "o"
1258
- ],
1259
- [
1260
- "e",
1261
- "e"
1262
- ],
1263
- [
1264
- "l",
1265
- "it"
1266
- ],
1267
- [
1268
- "t",
1269
- "al"
1270
- ],
1271
- [
1272
- "rc",
1273
- "h"
1274
- ],
1275
- [
1276
- "n",
1277
- "o"
1278
- ],
1279
- [
1280
- "l",
1281
- "u"
1282
- ],
1283
- [
1284
- "e",
1285
- "y"
1286
- ],
1287
- [
1288
- "c",
1289
- "ia"
1290
- ],
1291
- [
1292
- "i",
1293
- "ty"
1294
- ],
1295
- [
1296
- "l",
1297
- "tur"
1298
- ],
1299
- [
1300
- "a",
1301
- "i"
1302
- ],
1303
- [
1304
- "a",
1305
- "v"
1306
- ],
1307
- [
1308
- "n",
1309
- "al"
1310
- ],
1311
- [
1312
- "ur",
1313
- "e"
1314
- ],
1315
- [
1316
- "m",
1317
- "al"
1318
- ],
1319
- [
1320
- "u",
1321
- "m"
1322
- ],
1323
- [
1324
- "g",
1325
- "u"
1326
- ],
1327
- [
1328
- "t",
1329
- "er"
1330
- ],
1331
- [
1332
- "i",
1333
- "m"
1334
- ],
1335
- [
1336
- "a",
1337
- "ti"
1338
- ],
1339
- [
1340
- "s",
1341
- "o"
1342
- ],
1343
- [
1344
- "l",
1345
- "ar"
1346
- ],
1347
- [
1348
- "l",
1349
- "em"
1350
- ],
1351
- [
1352
- "c",
1353
- "er"
1354
- ],
1355
- [
1356
- "ti",
1357
- "on"
1358
- ],
1359
- [
1360
- "r",
1361
- "y"
1362
- ],
1363
- [
1364
- "m",
1365
- "at"
1366
- ],
1367
- [
1368
- "r",
1369
- "ce"
1370
- ],
1371
- [
1372
- "n",
1373
- "s"
1374
- ],
1375
- [
1376
- "a",
1377
- "te"
1378
- ],
1379
- [
1380
- "me",
1381
- "nt"
1382
- ],
1383
- [
1384
- "o",
1385
- "s"
1386
- ],
1387
- [
1388
- "e",
1389
- "ra"
1390
- ],
1391
- [
1392
- "t",
1393
- "or"
1394
- ],
1395
- [
1396
- "b",
1397
- "le"
1398
- ],
1399
- [
1400
- "e",
1401
- "k"
1402
- ],
1403
- [
1404
- "n",
1405
- "k"
1406
- ],
1407
- [
1408
- "m",
1409
- "i"
1410
- ],
1411
- [
1412
- "w",
1413
- "er"
1414
- ],
1415
- [
1416
- "e",
1417
- "ther"
1418
- ],
1419
- [
1420
- "e",
1421
- "en"
1422
- ],
1423
- [
1424
- "v",
1425
- "a"
1426
- ],
1427
- [
1428
- "o",
1429
- "us"
1430
- ],
1431
- [
1432
- "d",
1433
- "u"
1434
- ],
1435
- [
1436
- "h",
1437
- "in"
1438
- ],
1439
- [
1440
- "t",
1441
- "io"
1442
- ],
1443
- [
1444
- "i",
1445
- "al"
1446
- ],
1447
- [
1448
- "o",
1449
- "re"
1450
- ],
1451
- [
1452
- "e",
1453
- "as"
1454
- ],
1455
- [
1456
- "b",
1457
- "r"
1458
- ],
1459
- [
1460
- "d",
1461
- "er"
1462
- ],
1463
- [
1464
- "a",
1465
- "in"
1466
- ],
1467
- [
1468
- "o",
1469
- "ut"
1470
- ],
1471
- [
1472
- "f",
1473
- "i"
1474
- ],
1475
- [
1476
- "d",
1477
- "ing"
1478
- ],
1479
- [
1480
- "r",
1481
- "s"
1482
- ],
1483
- [
1484
- "a",
1485
- "tion"
1486
- ],
1487
- [
1488
- "i",
1489
- "at"
1490
- ],
1491
- [
1492
- "a",
1493
- "nt"
1494
- ],
1495
- [
1496
- "o",
1497
- "t"
1498
- ],
1499
- [
1500
- "a",
1501
- "re"
1502
- ],
1503
- [
1504
- "l",
1505
- "op"
1506
- ],
1507
- [
1508
- "t",
1509
- "p"
1510
- ],
1511
- [
1512
- "s",
1513
- "tem"
1514
- ],
1515
- [
1516
- "o",
1517
- "od"
1518
- ],
1519
- [
1520
- "u",
1521
- "en"
1522
- ],
1523
- [
1524
- "a",
1525
- "rch"
1526
- ],
1527
- [
1528
- "o",
1529
- "ol"
1530
- ]
1531
- ]
1532
- }
1533
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fw57M_Entropy_thresholdM_500/tokenizer_config.json DELETED
@@ -1,29 +0,0 @@
1
- {
2
- "add_prefix_space": true,
3
- "added_tokens_decoder": {
4
- "0": {
5
- "content": "<|padding|>",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": true
11
- },
12
- "1": {
13
- "content": "<|endoftext|>",
14
- "lstrip": false,
15
- "normalized": false,
16
- "rstrip": false,
17
- "single_word": false,
18
- "special": true
19
- }
20
- },
21
- "bos_token": "<|endoftext|>",
22
- "clean_up_tokenization_spaces": false,
23
- "eos_token": "<|endoftext|>",
24
- "extra_special_tokens": {},
25
- "model_max_length": 1000000000000000019884624838656,
26
- "pad_token": "<|padding|>",
27
- "tokenizer_class": "PreTrainedTokenizer",
28
- "unk_token": null
29
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fw57M_Entropy_thresholdM_500/vocab.json DELETED
@@ -1 +0,0 @@
1
- {"m": 78, "c": 68, "V": 55, "z": 91, "\u00b1": 111, "\u00f3": 177, "!": 2, "M": 46, "\u00a8": 103, "\u00d9": 151, "%": 6, "\u00ba": 120, "\u00de": 156, "\u0120": 222, "\u0137": 245, "\u013b": 249, "\u0143": 257, "\u00b6": 116, "\u00cc": 138, ";": 28, "Z": 59, "\u0132": 240, "e": 70, "[": 60, "\u00fe": 188, "\u00c5": 131, "0": 17, "\u0103": 193, "l": 77, "\u0111": 207, "\u011f": 221, "h": 73, "\u0122": 224, "\u0136": 244, "\u011c": 218, "\u0142": 256, ":": 27, "~": 95, "Q": 50, "\u00d4": 146, "\u00a3": 98, "\u00fb": 185, "\u00fc": 186, "\u010c": 202, "\u00e7": 165, "\u0139": 247, "\u010b": 201, "?": 32, "(": 9, "T": 53, "U": 54, "\u0121": 223, ".": 15, "\u0110": 206, "7": 24, "\u00ed": 171, "\u00fd": 187, "\u00ac": 107, "H": 41, "t": 85, "\u00ce": 140, "\u012d": 235, "\u012e": 236, "\u00cd": 139, "\u00c6": 132, "\u00d0": 142, "\u00bc": 122, "_": 64, "\"": 3, "\u00ec": 170, "5": 22, "\u0112": 208, "\u013c": 250, "<|padding|>": 0, "\\": 61, "\u0116": 212, "G": 40, "\u0124": 226, "|": 93, "-": 14, "A": 34, "\u00f6": 180, "\u00c0": 126, "\u00e0": 158, "\u0119": 215, "\u0133": 241, "\u00bb": 121, "K": 44, "j": 75, "\u00a6": 101, "\u00dd": 155, "+": 12, "\u00c7": 133, "\u0118": 214, "\u0125": 227, "\u0117": 213, "\u00e8": 166, "\u0134": 242, "q": 82, "\u0105": 195, "\u00e6": 164, "$": 5, "\u00bd": 123, "\u00f2": 176, "\u00a2": 97, "\u011a": 216, "\u00b4": 114, "\u00aa": 105, "\u00b9": 119, "\u011b": 217, "4": 21, "O": 48, "\u00d8": 150, "'": 8, "\u013a": 248, "J": 43, "\u013e": 252, "\u0108": 198, "\u010d": 203, "v": 87, "@": 33, "\u0107": 197, "1": 18, "/": 16, "\u013f": 253, "=": 30, "Y": 58, "\u010e": 204, "\u0109": 199, "\u0114": 210, "\u00be": 124, "\u011e": 220, "\u00d7": 149, "\u00b5": 115, "\u00f5": 179, "{": 92, "R": 51, "6": 23, "p": 81, "\u00f9": 183, "\u00d1": 143, "F": 39, ">": 31, "\u00a9": 104, "\u00e2": 160, "\u00a4": 99, "\u013d": 251, "\u012b": 233, "\u00fa": 184, "i": 74, "\u00b2": 112, "9": 26, "\u00ea": 168, "\u012f": 237, "\u00bf": 125, "\u0127": 229, "\u00a5": 100, "\u0130": 238, "\u00a7": 102, "\u00d5": 147, "\u00f7": 181, "*": 11, "\u0135": 243, "n": 79, "\u00af": 109, "\u00f0": 174, "\u00ff": 189, "}": 94, "x": 89, "\u00e4": 162, "B": 35, "\u00c9": 135, "\u00da": 152, "N": 47, "\u00ee": 172, "\u00f8": 182, "W": 56, "w": 88, "&": 7, "E": 38, "\u0100": 190, "\u010f": 205, "\u0128": 230, "\u0141": 255, "\u010a": 200, "\u0104": 194, "S": 52, "f": 71, "C": 36, "I": 42, "\u00db": 153, "\u0113": 209, "\u00c4": 130, "\u0129": 231, "\u0106": 196, "\u00dc": 154, ")": 10, "^": 63, "\u00d2": 144, "\u00eb": 169, "\u0126": 228, ",": 13, "\u00b0": 110, "r": 83, "<": 29, "\u00f1": 175, "a": 66, "\u00ca": 136, "k": 76, "\u00e3": 161, "\u0140": 254, "L": 45, "P": 49, "\u00d3": 145, "\u00b7": 117, "\u00f4": 178, "2": 19, "<|endoftext|>": 1, "\u00ae": 108, "\u00b3": 113, "\u00df": 157, "g": 72, "\u0102": 192, "\u0138": 246, "u": 86, "D": 37, "\u011d": 219, "\u0131": 239, "s": 84, "d": 69, "b": 67, "\u00cb": 137, "3": 20, "\u0101": 191, "\u00ef": 173, "\u012c": 234, "\u00e1": 159, "\u0115": 211, "\u00b8": 118, "\u00a1": 96, "\u00ab": 106, "\u00d6": 148, "\u00e9": 167, "X": 57, "y": 90, "\u00c1": 127, "\u00c3": 129, "\u00cf": 141, "#": 4, "\u00c8": 134, "8": 25, "\u00e5": 163, "`": 65, "o": 80, "\u0123": 225, "]": 62, "\u012a": 232, "\u00c2": 128, "ng": 258, "ge": 259, "he": 260, "ur": 261, "en": 262, "ve": 263, "er": 264, "nt": 265, "et": 266, "gh": 267, "ig": 268, "ss": 269, "me": 270, "ar": 271, "ing": 272, "on": 273, "rd": 274, "nd": 275, "dy": 276, "le": 277, "de": 278, "ll": 279, "ly": 280, "og": 281, "in": 282, "her": 283, "den": 284, "te": 285, "cl": 286, "li": 287, "ra": 288, "ch": 289, "ou": 290, "em": 291, "ke": 292, "io": 293, "th": 294, "ag": 295, "ct": 296, "at": 297, "an": 298, "rc": 299, "es": 300, "se": 301, "or": 302, "ut": 303, "gn": 304, "al": 305, "ea": 306, "ed": 307, "ow": 308, "ul": 309, "lt": 310, "the": 311, "us": 312, "pl": 313, "un": 314, "ce": 315, "ent": 316, "tur": 317, "ne": 318, "el": 319, "ec": 320, "ht": 321, "il": 322, "re": 323, "ld": 324, "tu": 325, "rg": 326, "rt": 327, "ro": 328, "ud": 329, "ess": 330, "ard": 331, "ri": 332, "ad": 333, "rv": 334, "st": 335, "am": 336, "om": 337, "sh": 338, "ti": 339, "tr": 340, "ple": 341, "ia": 342, "ap": 343, "ect": 344, "qu": 345, "use": 346, "ta": 347, "ay": 348, "ga": 349, "ver": 350, "and": 351, "is": 352, "ic": 353, "ho": 354, "ue": 355, "ion": 356, "id": 357, "lf": 358, "bl": 359, "oy": 360, "ha": 361, "nce": 362, "ma": 363, "iv": 364, "it": 365, "ug": 366, "mp": 367, "mb": 368, "hi": 369, "but": 370, "na": 371, "as": 372, "up": 373, "cu": 374, "ua": 375, "to": 376, "ult": 377, "ph": 378, "ck": 379, "tem": 380, "rat": 381, "ac": 382, "po": 383, "we": 384, "od": 385, "nc": 386, "rn": 387, "vi": 388, "sk": 389, "ear": 390, "di": 391, "do": 392, "op": 393, "uc": 394, "la": 395, "ka": 396, "ca": 397, "ui": 398, "ber": 399, "ith": 400, "rk": 401, "pt": 402, "ni": 403, "be": 404, "su": 405, "est": 406, "ther": 407, "ip": 408, "ci": 409, "ish": 410, "age": 411, "si": 412, "ol": 413, "ase": 414, "ugh": 415, "ty": 416, "rm": 417, "ine": 418, "ir": 419, "lo": 420, "und": 421, "ady": 422, "pa": 423, "men": 424, "lly": 425, "ght": 426, "ab": 427, "fe": 428, "ib": 429, "ork": 430, "oo": 431, "ee": 432, "lit": 433, "tal": 434, "rch": 435, "no": 436, "lu": 437, "ey": 438, "cia": 439, "ity": 440, "ltur": 441, "ai": 442, "av": 443, "nal": 444, "ure": 445, "mal": 446, "um": 447, "gu": 448, "ter": 449, "im": 450, "ati": 451, "so": 452, "lar": 453, "lem": 454, "cer": 455, "tion": 456, "ry": 457, "mat": 458, "rce": 459, "ns": 460, "ate": 461, "ment": 462, "os": 463, "era": 464, "tor": 465, "ble": 466, "ek": 467, "nk": 468, "mi": 469, "wer": 470, "ether": 471, "een": 472, "va": 473, "ous": 474, "du": 475, "hin": 476, "tio": 477, "ial": 478, "ore": 479, "eas": 480, "br": 481, "der": 482, "ain": 483, "out": 484, "fi": 485, "ding": 486, "rs": 487, "ation": 488, "iat": 489, "ant": 490, "ot": 491, "are": 492, "lop": 493, "tp": 494, "stem": 495, "ood": 496, "uen": 497, "arch": 498, "ool": 499}