Delete giga-tokens.txt

giga-tokens.txt

@@ -1,500 +0,0 @@
-<blk> 0
-<sos/eos> 1
-<unk> 2
-S 3
-T 4
-▁THE 5
-▁A 6
-E 7
-▁AND 8
-▁TO 9
-N 10
-D 11
-▁OF 12
-' 13
-ING 14
-▁I 15
-Y 16
-▁IN 17
-ED 18
-▁THAT 19
-▁ 20
-P 21
-R 22
-▁YOU 23
-M 24
-RE 25
-ER 26
-C 27
-O 28
-▁IT 29
-L 30
-A 31
-U 32
-G 33
-▁WE 34
-▁IS 35
-▁SO 36
-AL 37
-I 38
-▁S 39
-▁RE 40
-AR 41
-B 42
-▁FOR 43
-▁C 44
-▁BE 45
-LE 46
-F 47
-W 48
-▁E 49
-▁HE 50
-LL 51
-▁WAS 52
-LY 53
-OR 54
-IN 55
-▁F 56
-VE 57
-▁THIS 58
-TH 59
-K 60
-▁ON 61
-IT 62
-▁B 63
-▁WITH 64
-▁BUT 65
-EN 66
-CE 67
-RI 68
-▁DO 69
-UR 70
-▁HAVE 71
-▁DE 72
-▁ME 73
-▁T 74
-ENT 75
-CH 76
-▁THEY 77
-▁NOT 78
-ES 79
-V 80
-▁AS 81
-RA 82
-▁P 83
-ON 84
-TER 85
-▁ARE 86
-▁WHAT 87
-IC 88
-▁ST 89
-▁LIKE 90
-ATION 91
-▁OR 92
-▁CA 93
-▁AT 94
-H 95
-▁KNOW 96
-▁G 97
-AN 98
-▁CON 99
-IL 100
-ND 101
-RO 102
-▁HIS 103
-▁CAN 104
-▁ALL 105
-TE 106
-▁THERE 107
-▁SU 108
-▁MO 109
-▁MA 110
-LI 111
-▁ONE 112
-▁ABOUT 113
-LA 114
-▁CO 115
-- 116
-▁MY 117
-▁HAD 118
-CK 119
-NG 120
-▁NO 121
-MENT 122
-AD 123
-LO 124
-ME 125
-▁AN 126
-▁FROM 127
-NE 128
-▁IF 129
-VER 130
-▁JUST 131
-▁PRO 132
-ION 133
-▁PA 134
-▁WHO 135
-▁SE 136
-EL 137
-IR 138
-▁US 139
-▁UP 140
-▁YOUR 141
-CI 142
-RY 143
-▁GO 144
-▁SHE 145
-▁LE 146
-▁OUT 147
-▁PO 148
-▁HO 149
-ATE 150
-▁BO 151
-▁BY 152
-▁FA 153
-▁MI 154
-AS 155
-MP 156
-▁HER 157
-VI 158
-▁THINK 159
-▁SOME 160
-▁WHEN 161
-▁AH 162
-▁PEOPLE 163
-IG 164
-▁WA 165
-▁TE 166
-▁LA 167
-▁WERE 168
-▁LI 169
-▁WOULD 170
-▁SEE 171
-▁WHICH 172
-DE 173
-GE 174
-▁K 175
-IGHT 176
-▁HA 177
-▁OUR 178
-UN 179
-▁HOW 180
-▁GET 181
-IS 182
-UT 183
-Z 184
-CO 185
-ET 186
-UL 187
-IES 188
-IVE 189
-AT 190
-▁O 191
-▁DON 192
-LU 193
-▁TIME 194
-▁WILL 195
-▁MORE 196
-▁SP 197
-▁NOW 198
-RU 199
-▁THEIR 200
-▁UN 201
-ITY 202
-OL 203
-X 204
-TI 205
-US 206
-▁VERY 207
-TION 208
-▁FI 209
-▁SAY 210
-▁BECAUSE 211
-▁EX 212
-▁RO 213
-ERS 214
-IST 215
-▁DA 216
-TING 217
-▁EN 218
-OM 219
-▁BA 220
-▁BEEN 221
-▁LO 222
-▁UM 223
-AGE 224
-ABLE 225
-▁WO 226
-▁RA 227
-▁OTHER 228
-▁REALLY 229
-ENCE 230
-▁GOING 231
-▁HIM 232
-▁HAS 233
-▁THEM 234
-▁DIS 235
-▁WANT 236
-ID 237
-TA 238
-▁LOOK 239
-KE 240
-▁DID 241
-▁SA 242
-▁VI 243
-▁SAID 244
-▁RIGHT 245
-▁THESE 246
-▁WORK 247
-▁COM 248
-ALLY 249
-FF 250
-QU 251
-AC 252
-▁DR 253
-▁WAY 254
-▁INTO 255
-MO 256
-TED 257
-EST 258
-▁HERE 259
-OK 260
-▁COULD 261
-▁WELL 262
-MA 263
-▁PRE 264
-▁DI 265
-MAN 266
-▁COMP 267
-▁THEN 268
-IM 269
-▁PER 270
-▁NA 271
-▁WHERE 272
-▁TWO 273
-▁WI 274
-▁FE 275
-INE 276
-▁ANY 277
-TURE 278
-▁OVER 279
-BO 280
-ACH 281
-OW 282
-▁MAKE 283
-▁TRA 284
-HE 285
-UND 286
-▁EVEN 287
-ANCE 288
-▁YEAR 289
-HO 290
-AM 291
-▁CHA 292
-▁BACK 293
-VO 294
-ANT 295
-DI 296
-▁ALSO 297
-▁THOSE 298
-▁MAN 299
-CTION 300
-ICAL 301
-▁JO 302
-▁OP 303
-▁NEW 304
-▁MU 305
-▁HU 306
-▁KIND 307
-▁NE 308
-CA 309
-END 310
-TIC 311
-FUL 312
-▁YEAH 313
-SH 314
-▁APP 315
-▁THINGS 316
-SIDE 317
-▁GOOD 318
-ONE 319
-▁TAKE 320
-CU 321
-▁EVERY 322
-▁MEAN 323
-▁FIRST 324
-OP 325
-▁TH 326
-▁MUCH 327
-▁PART 328
-UGH 329
-▁COME 330
-J 331
-▁THAN 332
-▁EXP 333
-▁AGAIN 334
-▁LITTLE 335
-MB 336
-▁NEED 337
-▁TALK 338
-IF 339
-FOR 340
-▁SH 341
-ISH 342
-▁STA 343
-ATED 344
-▁GU 345
-▁LET 346
-IA 347
-▁MAR 348
-▁DOWN 349
-▁DAY 350
-▁GA 351
-▁SOMETHING 352
-▁BU 353
-DUC 354
-HA 355
-▁LOT 356
-▁RU 357
-▁THOUGH 358
-▁GREAT 359
-AIN 360
-▁THROUGH 361
-▁THING 362
-OUS 363
-▁PRI 364
-▁GOT 365
-▁SHOULD 366
-▁AFTER 367
-▁HEAR 368
-▁TA 369
-▁ONLY 370
-▁CHI 371
-IOUS 372
-▁SHA 373
-▁MOST 374
-▁ACTUALLY 375
-▁START 376
-LIC 377
-▁VA 378
-▁RI 379
-DAY 380
-IAN 381
-▁DOES 382
-ROW 383
-▁GRA 384
-ITION 385
-▁MANY 386
-▁BEFORE 387
-▁GIVE 388
-PORT 389
-QUI 390
-▁LIFE 391
-▁WORLD 392
-▁PI 393
-▁LONG 394
-▁THREE 395
-IZE 396
-NESS 397
-▁SHOW 398
-PH 399
-▁WHY 400
-▁QUESTION 401
-WARD 402
-▁THANK 403
-▁PH 404
-▁DIFFERENT 405
-▁OWN 406
-▁FEEL 407
-▁MIGHT 408
-▁HAPPEN 409
-▁MADE 410
-▁BRO 411
-IBLE 412
-▁HI 413
-▁STATE 414
-▁HAND 415
-▁NEVER 416
-▁PLACE 417
-▁LOVE 418
-▁DU 419
-▁POINT 420
-▁HELP 421
-▁COUNT 422
-▁STILL 423
-▁MR 424
-▁FIND 425
-▁PERSON 426
-▁CAME 427
-▁SAME 428
-▁LAST 429
-▁HIGH 430
-▁OLD 431
-▁UNDER 432
-▁FOUR 433
-▁AROUND 434
-▁SORT 435
-▁CHANGE 436
-▁YES 437
-SHIP 438
-▁ANOTHER 439
-ATIVE 440
-▁FOUND 441
-▁JA 442
-▁ALWAYS 443
-▁NEXT 444
-▁TURN 445
-▁JU 446
-▁SIX 447
-▁FACT 448
-▁INTEREST 449
-▁WORD 450
-▁THOUSAND 451
-▁HUNDRED 452
-▁NUMBER 453
-▁IDEA 454
-▁PLAN 455
-▁COURSE 456
-▁SCHOOL 457
-▁HOUSE 458
-▁TWENTY 459
-▁JE 460
-▁PLAY 461
-▁AWAY 462
-▁LEARN 463
-▁HARD 464
-▁WEEK 465
-▁BETTER 466
-▁WHILE 467
-▁FRIEND 468
-▁OKAY 469
-▁NINE 470
-▁UNDERSTAND 471
-▁KEEP 472
-▁GONNA 473
-▁SYSTEM 474
-▁AMERICA 475
-▁POWER 476
-▁IMPORTANT 477
-▁WITHOUT 478
-▁MAYBE 479
-▁SEVEN 480
-▁BETWEEN 481
-▁BUILD 482
-▁CERTAIN 483
-▁PROBLEM 484
-▁MONEY 485
-▁BELIEVE 486
-▁SECOND 487
-▁REASON 488
-▁TOGETHER 489
-▁PUBLIC 490
-▁ANYTHING 491
-▁SPEAK 492
-▁BUSINESS 493
-▁EVERYTHING 494
-▁CLOSE 495
-▁QUITE 496
-▁ANSWER 497
-▁ENOUGH 498
-Q 499