Hugging Face's logo

MarkGG
/
Romance-cleaned-1

Text Generation
Transformers
PyTorch
TensorFlow
TensorBoard
gpt2
Generated from Trainer
text-generation-inference
Model card Files
xet
 Metrics Community
1
MarkGG commited on
Commit
69764d9
·
1 Parent(s): f50505a

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +0 -1
  2. tokenizer.json +209 -214
  3. tokenizer_config.json +1 -1
special_tokens_map.json CHANGED
@@ -1,6 +1,5 @@
1
  {
2
  "bos_token": "<|endoftext|>",
3
  "eos_token": "<|endoftext|>",
4
- "pad_token": "<|endoftext|>",
5
  "unk_token": "<|endoftext|>"
6
  }
 
1
  {
2
  "bos_token": "<|endoftext|>",
3
  "eos_token": "<|endoftext|>",
 
4
  "unk_token": "<|endoftext|>"
5
  }
tokenizer.json CHANGED
@@ -1,11 +1,6 @@
1
  {
2
  "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 128,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
  "padding": null,
10
  "added_tokens": [
11
  {
@@ -23,8 +18,8 @@
23
  "single_word": false,
24
  "lstrip": false,
25
  "rstrip": false,
26
- "normalized": true,
27
- "special": false
28
  },
29
  {
30
  "id": 52001,
@@ -32,8 +27,8 @@
32
  "single_word": false,
33
  "lstrip": false,
34
  "rstrip": false,
35
- "normalized": true,
36
- "special": false
37
  },
38
  {
39
  "id": 52002,
@@ -41,8 +36,8 @@
41
  "single_word": false,
42
  "lstrip": false,
43
  "rstrip": false,
44
- "normalized": true,
45
- "special": false
46
  },
47
  {
48
  "id": 52003,
@@ -50,8 +45,8 @@
50
  "single_word": false,
51
  "lstrip": false,
52
  "rstrip": false,
53
- "normalized": true,
54
- "special": false
55
  },
56
  {
57
  "id": 52004,
@@ -59,8 +54,8 @@
59
  "single_word": false,
60
  "lstrip": false,
61
  "rstrip": false,
62
- "normalized": true,
63
- "special": false
64
  },
65
  {
66
  "id": 52005,
@@ -68,8 +63,8 @@
68
  "single_word": false,
69
  "lstrip": false,
70
  "rstrip": false,
71
- "normalized": true,
72
- "special": false
73
  },
74
  {
75
  "id": 52006,
@@ -77,8 +72,8 @@
77
  "single_word": false,
78
  "lstrip": false,
79
  "rstrip": false,
80
- "normalized": true,
81
- "special": false
82
  },
83
  {
84
  "id": 52007,
@@ -86,8 +81,8 @@
86
  "single_word": false,
87
  "lstrip": false,
88
  "rstrip": false,
89
- "normalized": true,
90
- "special": false
91
  },
92
  {
93
  "id": 52008,
@@ -95,8 +90,8 @@
95
  "single_word": false,
96
  "lstrip": false,
97
  "rstrip": false,
98
- "normalized": true,
99
- "special": false
100
  },
101
  {
102
  "id": 52009,
@@ -104,8 +99,8 @@
104
  "single_word": false,
105
  "lstrip": false,
106
  "rstrip": false,
107
- "normalized": true,
108
- "special": false
109
  },
110
  {
111
  "id": 52010,
@@ -113,8 +108,8 @@
113
  "single_word": false,
114
  "lstrip": false,
115
  "rstrip": false,
116
- "normalized": true,
117
- "special": false
118
  },
119
  {
120
  "id": 52011,
@@ -122,8 +117,8 @@
122
  "single_word": false,
123
  "lstrip": false,
124
  "rstrip": false,
125
- "normalized": true,
126
- "special": false
127
  },
128
  {
129
  "id": 52012,
@@ -131,8 +126,8 @@
131
  "single_word": false,
132
  "lstrip": false,
133
  "rstrip": false,
134
- "normalized": true,
135
- "special": false
136
  },
137
  {
138
  "id": 52013,
@@ -140,8 +135,8 @@
140
  "single_word": false,
141
  "lstrip": false,
142
  "rstrip": false,
143
- "normalized": true,
144
- "special": false
145
  },
146
  {
147
  "id": 52014,
@@ -149,8 +144,8 @@
149
  "single_word": false,
150
  "lstrip": false,
151
  "rstrip": false,
152
- "normalized": true,
153
- "special": false
154
  },
155
  {
156
  "id": 52015,
@@ -158,8 +153,8 @@
158
  "single_word": false,
159
  "lstrip": false,
160
  "rstrip": false,
161
- "normalized": true,
162
- "special": false
163
  },
164
  {
165
  "id": 52016,
@@ -167,8 +162,8 @@
167
  "single_word": false,
168
  "lstrip": false,
169
  "rstrip": false,
170
- "normalized": true,
171
- "special": false
172
  },
173
  {
174
  "id": 52017,
@@ -176,8 +171,8 @@
176
  "single_word": false,
177
  "lstrip": false,
178
  "rstrip": false,
179
- "normalized": true,
180
- "special": false
181
  },
182
  {
183
  "id": 52018,
@@ -185,8 +180,8 @@
185
  "single_word": false,
186
  "lstrip": false,
187
  "rstrip": false,
188
- "normalized": true,
189
- "special": false
190
  },
191
  {
192
  "id": 52019,
@@ -194,8 +189,8 @@
194
  "single_word": false,
195
  "lstrip": false,
196
  "rstrip": false,
197
- "normalized": true,
198
- "special": false
199
  },
200
  {
201
  "id": 52020,
@@ -203,8 +198,8 @@
203
  "single_word": false,
204
  "lstrip": false,
205
  "rstrip": false,
206
- "normalized": true,
207
- "special": false
208
  },
209
  {
210
  "id": 52021,
@@ -212,8 +207,8 @@
212
  "single_word": false,
213
  "lstrip": false,
214
  "rstrip": false,
215
- "normalized": true,
216
- "special": false
217
  },
218
  {
219
  "id": 52022,
@@ -221,8 +216,8 @@
221
  "single_word": false,
222
  "lstrip": false,
223
  "rstrip": false,
224
- "normalized": true,
225
- "special": false
226
  },
227
  {
228
  "id": 52023,
@@ -230,8 +225,8 @@
230
  "single_word": false,
231
  "lstrip": false,
232
  "rstrip": false,
233
- "normalized": true,
234
- "special": false
235
  },
236
  {
237
  "id": 52024,
@@ -239,8 +234,8 @@
239
  "single_word": false,
240
  "lstrip": false,
241
  "rstrip": false,
242
- "normalized": true,
243
- "special": false
244
  },
245
  {
246
  "id": 52025,
@@ -248,8 +243,8 @@
248
  "single_word": false,
249
  "lstrip": false,
250
  "rstrip": false,
251
- "normalized": true,
252
- "special": false
253
  },
254
  {
255
  "id": 52026,
@@ -257,8 +252,8 @@
257
  "single_word": false,
258
  "lstrip": false,
259
  "rstrip": false,
260
- "normalized": true,
261
- "special": false
262
  },
263
  {
264
  "id": 52027,
@@ -266,8 +261,8 @@
266
  "single_word": false,
267
  "lstrip": false,
268
  "rstrip": false,
269
- "normalized": true,
270
- "special": false
271
  },
272
  {
273
  "id": 52028,
@@ -275,8 +270,8 @@
275
  "single_word": false,
276
  "lstrip": false,
277
  "rstrip": false,
278
- "normalized": true,
279
- "special": false
280
  },
281
  {
282
  "id": 52029,
@@ -284,8 +279,8 @@
284
  "single_word": false,
285
  "lstrip": false,
286
  "rstrip": false,
287
- "normalized": true,
288
- "special": false
289
  },
290
  {
291
  "id": 52030,
@@ -293,8 +288,8 @@
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
296
- "normalized": true,
297
- "special": false
298
  },
299
  {
300
  "id": 52031,
@@ -302,8 +297,8 @@
302
  "single_word": false,
303
  "lstrip": false,
304
  "rstrip": false,
305
- "normalized": true,
306
- "special": false
307
  },
308
  {
309
  "id": 52032,
@@ -311,8 +306,8 @@
311
  "single_word": false,
312
  "lstrip": false,
313
  "rstrip": false,
314
- "normalized": true,
315
- "special": false
316
  },
317
  {
318
  "id": 52033,
@@ -320,8 +315,8 @@
320
  "single_word": false,
321
  "lstrip": false,
322
  "rstrip": false,
323
- "normalized": true,
324
- "special": false
325
  },
326
  {
327
  "id": 52034,
@@ -329,8 +324,8 @@
329
  "single_word": false,
330
  "lstrip": false,
331
  "rstrip": false,
332
- "normalized": true,
333
- "special": false
334
  },
335
  {
336
  "id": 52035,
@@ -338,8 +333,8 @@
338
  "single_word": false,
339
  "lstrip": false,
340
  "rstrip": false,
341
- "normalized": true,
342
- "special": false
343
  },
344
  {
345
  "id": 52036,
@@ -347,8 +342,8 @@
347
  "single_word": false,
348
  "lstrip": false,
349
  "rstrip": false,
350
- "normalized": true,
351
- "special": false
352
  },
353
  {
354
  "id": 52037,
@@ -356,8 +351,8 @@
356
  "single_word": false,
357
  "lstrip": false,
358
  "rstrip": false,
359
- "normalized": true,
360
- "special": false
361
  },
362
  {
363
  "id": 52038,
@@ -365,8 +360,8 @@
365
  "single_word": false,
366
  "lstrip": false,
367
  "rstrip": false,
368
- "normalized": true,
369
- "special": false
370
  },
371
  {
372
  "id": 52039,
@@ -374,8 +369,8 @@
374
  "single_word": false,
375
  "lstrip": false,
376
  "rstrip": false,
377
- "normalized": true,
378
- "special": false
379
  },
380
  {
381
  "id": 52040,
@@ -383,8 +378,8 @@
383
  "single_word": false,
384
  "lstrip": false,
385
  "rstrip": false,
386
- "normalized": true,
387
- "special": false
388
  },
389
  {
390
  "id": 52041,
@@ -392,8 +387,8 @@
392
  "single_word": false,
393
  "lstrip": false,
394
  "rstrip": false,
395
- "normalized": true,
396
- "special": false
397
  },
398
  {
399
  "id": 52042,
@@ -401,8 +396,8 @@
401
  "single_word": false,
402
  "lstrip": false,
403
  "rstrip": false,
404
- "normalized": true,
405
- "special": false
406
  },
407
  {
408
  "id": 52043,
@@ -410,8 +405,8 @@
410
  "single_word": false,
411
  "lstrip": false,
412
  "rstrip": false,
413
- "normalized": true,
414
- "special": false
415
  },
416
  {
417
  "id": 52044,
@@ -419,8 +414,8 @@
419
  "single_word": false,
420
  "lstrip": false,
421
  "rstrip": false,
422
- "normalized": true,
423
- "special": false
424
  },
425
  {
426
  "id": 52045,
@@ -428,8 +423,8 @@
428
  "single_word": false,
429
  "lstrip": false,
430
  "rstrip": false,
431
- "normalized": true,
432
- "special": false
433
  },
434
  {
435
  "id": 52046,
@@ -437,8 +432,8 @@
437
  "single_word": false,
438
  "lstrip": false,
439
  "rstrip": false,
440
- "normalized": true,
441
- "special": false
442
  },
443
  {
444
  "id": 52047,
@@ -446,8 +441,8 @@
446
  "single_word": false,
447
  "lstrip": false,
448
  "rstrip": false,
449
- "normalized": true,
450
- "special": false
451
  },
452
  {
453
  "id": 52048,
@@ -455,8 +450,8 @@
455
  "single_word": false,
456
  "lstrip": false,
457
  "rstrip": false,
458
- "normalized": true,
459
- "special": false
460
  },
461
  {
462
  "id": 52049,
@@ -464,8 +459,8 @@
464
  "single_word": false,
465
  "lstrip": false,
466
  "rstrip": false,
467
- "normalized": true,
468
- "special": false
469
  },
470
  {
471
  "id": 52050,
@@ -473,8 +468,8 @@
473
  "single_word": false,
474
  "lstrip": false,
475
  "rstrip": false,
476
- "normalized": true,
477
- "special": false
478
  },
479
  {
480
  "id": 52051,
@@ -482,8 +477,8 @@
482
  "single_word": false,
483
  "lstrip": false,
484
  "rstrip": false,
485
- "normalized": true,
486
- "special": false
487
  },
488
  {
489
  "id": 52052,
@@ -491,8 +486,8 @@
491
  "single_word": false,
492
  "lstrip": false,
493
  "rstrip": false,
494
- "normalized": true,
495
- "special": false
496
  },
497
  {
498
  "id": 52053,
@@ -500,8 +495,8 @@
500
  "single_word": false,
501
  "lstrip": false,
502
  "rstrip": false,
503
- "normalized": true,
504
- "special": false
505
  },
506
  {
507
  "id": 52054,
@@ -509,8 +504,8 @@
509
  "single_word": false,
510
  "lstrip": false,
511
  "rstrip": false,
512
- "normalized": true,
513
- "special": false
514
  },
515
  {
516
  "id": 52055,
@@ -518,8 +513,8 @@
518
  "single_word": false,
519
  "lstrip": false,
520
  "rstrip": false,
521
- "normalized": true,
522
- "special": false
523
  },
524
  {
525
  "id": 52056,
@@ -527,8 +522,8 @@
527
  "single_word": false,
528
  "lstrip": false,
529
  "rstrip": false,
530
- "normalized": true,
531
- "special": false
532
  },
533
  {
534
  "id": 52057,
@@ -536,8 +531,8 @@
536
  "single_word": false,
537
  "lstrip": false,
538
  "rstrip": false,
539
- "normalized": true,
540
- "special": false
541
  },
542
  {
543
  "id": 52058,
@@ -545,8 +540,8 @@
545
  "single_word": false,
546
  "lstrip": false,
547
  "rstrip": false,
548
- "normalized": true,
549
- "special": false
550
  },
551
  {
552
  "id": 52059,
@@ -554,8 +549,8 @@
554
  "single_word": false,
555
  "lstrip": false,
556
  "rstrip": false,
557
- "normalized": true,
558
- "special": false
559
  },
560
  {
561
  "id": 52060,
@@ -563,8 +558,8 @@
563
  "single_word": false,
564
  "lstrip": false,
565
  "rstrip": false,
566
- "normalized": true,
567
- "special": false
568
  },
569
  {
570
  "id": 52061,
@@ -572,8 +567,8 @@
572
  "single_word": false,
573
  "lstrip": false,
574
  "rstrip": false,
575
- "normalized": true,
576
- "special": false
577
  },
578
  {
579
  "id": 52062,
@@ -581,8 +576,8 @@
581
  "single_word": false,
582
  "lstrip": false,
583
  "rstrip": false,
584
- "normalized": true,
585
- "special": false
586
  },
587
  {
588
  "id": 52063,
@@ -590,8 +585,8 @@
590
  "single_word": false,
591
  "lstrip": false,
592
  "rstrip": false,
593
- "normalized": true,
594
- "special": false
595
  },
596
  {
597
  "id": 52064,
@@ -599,8 +594,8 @@
599
  "single_word": false,
600
  "lstrip": false,
601
  "rstrip": false,
602
- "normalized": true,
603
- "special": false
604
  },
605
  {
606
  "id": 52065,
@@ -608,8 +603,8 @@
608
  "single_word": false,
609
  "lstrip": false,
610
  "rstrip": false,
611
- "normalized": true,
612
- "special": false
613
  },
614
  {
615
  "id": 52066,
@@ -617,8 +612,8 @@
617
  "single_word": false,
618
  "lstrip": false,
619
  "rstrip": false,
620
- "normalized": true,
621
- "special": false
622
  },
623
  {
624
  "id": 52067,
@@ -626,8 +621,8 @@
626
  "single_word": false,
627
  "lstrip": false,
628
  "rstrip": false,
629
- "normalized": true,
630
- "special": false
631
  },
632
  {
633
  "id": 52068,
@@ -635,8 +630,8 @@
635
  "single_word": false,
636
  "lstrip": false,
637
  "rstrip": false,
638
- "normalized": true,
639
- "special": false
640
  },
641
  {
642
  "id": 52069,
@@ -644,8 +639,8 @@
644
  "single_word": false,
645
  "lstrip": false,
646
  "rstrip": false,
647
- "normalized": true,
648
- "special": false
649
  },
650
  {
651
  "id": 52070,
@@ -653,8 +648,8 @@
653
  "single_word": false,
654
  "lstrip": false,
655
  "rstrip": false,
656
- "normalized": true,
657
- "special": false
658
  },
659
  {
660
  "id": 52071,
@@ -662,8 +657,8 @@
662
  "single_word": false,
663
  "lstrip": false,
664
  "rstrip": false,
665
- "normalized": true,
666
- "special": false
667
  },
668
  {
669
  "id": 52072,
@@ -671,8 +666,8 @@
671
  "single_word": false,
672
  "lstrip": false,
673
  "rstrip": false,
674
- "normalized": true,
675
- "special": false
676
  },
677
  {
678
  "id": 52073,
@@ -680,8 +675,8 @@
680
  "single_word": false,
681
  "lstrip": false,
682
  "rstrip": false,
683
- "normalized": true,
684
- "special": false
685
  },
686
  {
687
  "id": 52074,
@@ -689,8 +684,8 @@
689
  "single_word": false,
690
  "lstrip": false,
691
  "rstrip": false,
692
- "normalized": true,
693
- "special": false
694
  },
695
  {
696
  "id": 52075,
@@ -698,8 +693,8 @@
698
  "single_word": false,
699
  "lstrip": false,
700
  "rstrip": false,
701
- "normalized": true,
702
- "special": false
703
  },
704
  {
705
  "id": 52076,
@@ -707,8 +702,8 @@
707
  "single_word": false,
708
  "lstrip": false,
709
  "rstrip": false,
710
- "normalized": true,
711
- "special": false
712
  },
713
  {
714
  "id": 52077,
@@ -716,8 +711,8 @@
716
  "single_word": false,
717
  "lstrip": false,
718
  "rstrip": false,
719
- "normalized": true,
720
- "special": false
721
  },
722
  {
723
  "id": 52078,
@@ -725,8 +720,8 @@
725
  "single_word": false,
726
  "lstrip": false,
727
  "rstrip": false,
728
- "normalized": true,
729
- "special": false
730
  },
731
  {
732
  "id": 52079,
@@ -734,8 +729,8 @@
734
  "single_word": false,
735
  "lstrip": false,
736
  "rstrip": false,
737
- "normalized": true,
738
- "special": false
739
  },
740
  {
741
  "id": 52080,
@@ -743,8 +738,8 @@
743
  "single_word": false,
744
  "lstrip": false,
745
  "rstrip": false,
746
- "normalized": true,
747
- "special": false
748
  },
749
  {
750
  "id": 52081,
@@ -752,8 +747,8 @@
752
  "single_word": false,
753
  "lstrip": false,
754
  "rstrip": false,
755
- "normalized": true,
756
- "special": false
757
  },
758
  {
759
  "id": 52082,
@@ -761,8 +756,8 @@
761
  "single_word": false,
762
  "lstrip": false,
763
  "rstrip": false,
764
- "normalized": true,
765
- "special": false
766
  },
767
  {
768
  "id": 52083,
@@ -770,8 +765,8 @@
770
  "single_word": false,
771
  "lstrip": false,
772
  "rstrip": false,
773
- "normalized": true,
774
- "special": false
775
  },
776
  {
777
  "id": 52084,
@@ -779,8 +774,8 @@
779
  "single_word": false,
780
  "lstrip": false,
781
  "rstrip": false,
782
- "normalized": true,
783
- "special": false
784
  },
785
  {
786
  "id": 52085,
@@ -788,8 +783,8 @@
788
  "single_word": false,
789
  "lstrip": false,
790
  "rstrip": false,
791
- "normalized": true,
792
- "special": false
793
  },
794
  {
795
  "id": 52086,
@@ -797,8 +792,8 @@
797
  "single_word": false,
798
  "lstrip": false,
799
  "rstrip": false,
800
- "normalized": true,
801
- "special": false
802
  },
803
  {
804
  "id": 52087,
@@ -806,8 +801,8 @@
806
  "single_word": false,
807
  "lstrip": false,
808
  "rstrip": false,
809
- "normalized": true,
810
- "special": false
811
  },
812
  {
813
  "id": 52088,
@@ -815,8 +810,8 @@
815
  "single_word": false,
816
  "lstrip": false,
817
  "rstrip": false,
818
- "normalized": true,
819
- "special": false
820
  },
821
  {
822
  "id": 52089,
@@ -824,8 +819,8 @@
824
  "single_word": false,
825
  "lstrip": false,
826
  "rstrip": false,
827
- "normalized": true,
828
- "special": false
829
  },
830
  {
831
  "id": 52090,
@@ -833,8 +828,8 @@
833
  "single_word": false,
834
  "lstrip": false,
835
  "rstrip": false,
836
- "normalized": true,
837
- "special": false
838
  },
839
  {
840
  "id": 52091,
@@ -842,8 +837,8 @@
842
  "single_word": false,
843
  "lstrip": false,
844
  "rstrip": false,
845
- "normalized": true,
846
- "special": false
847
  },
848
  {
849
  "id": 52092,
@@ -851,8 +846,8 @@
851
  "single_word": false,
852
  "lstrip": false,
853
  "rstrip": false,
854
- "normalized": true,
855
- "special": false
856
  },
857
  {
858
  "id": 52093,
@@ -860,8 +855,8 @@
860
  "single_word": false,
861
  "lstrip": false,
862
  "rstrip": false,
863
- "normalized": true,
864
- "special": false
865
  },
866
  {
867
  "id": 52094,
@@ -869,8 +864,8 @@
869
  "single_word": false,
870
  "lstrip": false,
871
  "rstrip": false,
872
- "normalized": true,
873
- "special": false
874
  },
875
  {
876
  "id": 52095,
@@ -878,8 +873,8 @@
878
  "single_word": false,
879
  "lstrip": false,
880
  "rstrip": false,
881
- "normalized": true,
882
- "special": false
883
  },
884
  {
885
  "id": 52096,
@@ -887,8 +882,8 @@
887
  "single_word": false,
888
  "lstrip": false,
889
  "rstrip": false,
890
- "normalized": true,
891
- "special": false
892
  },
893
  {
894
  "id": 52097,
@@ -896,8 +891,8 @@
896
  "single_word": false,
897
  "lstrip": false,
898
  "rstrip": false,
899
- "normalized": true,
900
- "special": false
901
  },
902
  {
903
  "id": 52098,
@@ -905,8 +900,8 @@
905
  "single_word": false,
906
  "lstrip": false,
907
  "rstrip": false,
908
- "normalized": true,
909
- "special": false
910
  },
911
  {
912
  "id": 52099,
@@ -914,8 +909,8 @@
914
  "single_word": false,
915
  "lstrip": false,
916
  "rstrip": false,
917
- "normalized": true,
918
- "special": false
919
  },
920
  {
921
  "id": 52100,
@@ -923,8 +918,8 @@
923
  "single_word": false,
924
  "lstrip": false,
925
  "rstrip": false,
926
- "normalized": true,
927
- "special": false
928
  },
929
  {
930
  "id": 52101,
@@ -932,8 +927,8 @@
932
  "single_word": false,
933
  "lstrip": false,
934
  "rstrip": false,
935
- "normalized": true,
936
- "special": false
937
  },
938
  {
939
  "id": 52102,
@@ -941,8 +936,8 @@
941
  "single_word": false,
942
  "lstrip": false,
943
  "rstrip": false,
944
- "normalized": true,
945
- "special": false
946
  },
947
  {
948
  "id": 52103,
@@ -950,8 +945,8 @@
950
  "single_word": false,
951
  "lstrip": false,
952
  "rstrip": false,
953
- "normalized": true,
954
- "special": false
955
  }
956
  ],
957
  "normalizer": null,
 
1
  {
2
  "version": "1.0",
3
+ "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
 
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
  },
24
  {
25
  "id": 52001,
 
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
  },
33
  {
34
  "id": 52002,
 
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
  },
42
  {
43
  "id": 52003,
 
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
  },
51
  {
52
  "id": 52004,
 
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
  },
60
  {
61
  "id": 52005,
 
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
66
+ "normalized": false,
67
+ "special": true
68
  },
69
  {
70
  "id": 52006,
 
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": false,
75
+ "normalized": false,
76
+ "special": true
77
  },
78
  {
79
  "id": 52007,
 
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
84
+ "normalized": false,
85
+ "special": true
86
  },
87
  {
88
  "id": 52008,
 
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
93
+ "normalized": false,
94
+ "special": true
95
  },
96
  {
97
  "id": 52009,
 
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
102
+ "normalized": false,
103
+ "special": true
104
  },
105
  {
106
  "id": 52010,
 
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
111
+ "normalized": false,
112
+ "special": true
113
  },
114
  {
115
  "id": 52011,
 
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": false,
120
+ "normalized": false,
121
+ "special": true
122
  },
123
  {
124
  "id": 52012,
 
126
  "single_word": false,
127
  "lstrip": false,
128
  "rstrip": false,
129
+ "normalized": false,
130
+ "special": true
131
  },
132
  {
133
  "id": 52013,
 
135
  "single_word": false,
136
  "lstrip": false,
137
  "rstrip": false,
138
+ "normalized": false,
139
+ "special": true
140
  },
141
  {
142
  "id": 52014,
 
144
  "single_word": false,
145
  "lstrip": false,
146
  "rstrip": false,
147
+ "normalized": false,
148
+ "special": true
149
  },
150
  {
151
  "id": 52015,
 
153
  "single_word": false,
154
  "lstrip": false,
155
  "rstrip": false,
156
+ "normalized": false,
157
+ "special": true
158
  },
159
  {
160
  "id": 52016,
 
162
  "single_word": false,
163
  "lstrip": false,
164
  "rstrip": false,
165
+ "normalized": false,
166
+ "special": true
167
  },
168
  {
169
  "id": 52017,
 
171
  "single_word": false,
172
  "lstrip": false,
173
  "rstrip": false,
174
+ "normalized": false,
175
+ "special": true
176
  },
177
  {
178
  "id": 52018,
 
180
  "single_word": false,
181
  "lstrip": false,
182
  "rstrip": false,
183
+ "normalized": false,
184
+ "special": true
185
  },
186
  {
187
  "id": 52019,
 
189
  "single_word": false,
190
  "lstrip": false,
191
  "rstrip": false,
192
+ "normalized": false,
193
+ "special": true
194
  },
195
  {
196
  "id": 52020,
 
198
  "single_word": false,
199
  "lstrip": false,
200
  "rstrip": false,
201
+ "normalized": false,
202
+ "special": true
203
  },
204
  {
205
  "id": 52021,
 
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,
210
+ "normalized": false,
211
+ "special": true
212
  },
213
  {
214
  "id": 52022,
 
216
  "single_word": false,
217
  "lstrip": false,
218
  "rstrip": false,
219
+ "normalized": false,
220
+ "special": true
221
  },
222
  {
223
  "id": 52023,
 
225
  "single_word": false,
226
  "lstrip": false,
227
  "rstrip": false,
228
+ "normalized": false,
229
+ "special": true
230
  },
231
  {
232
  "id": 52024,
 
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
237
+ "normalized": false,
238
+ "special": true
239
  },
240
  {
241
  "id": 52025,
 
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
246
+ "normalized": false,
247
+ "special": true
248
  },
249
  {
250
  "id": 52026,
 
252
  "single_word": false,
253
  "lstrip": false,
254
  "rstrip": false,
255
+ "normalized": false,
256
+ "special": true
257
  },
258
  {
259
  "id": 52027,
 
261
  "single_word": false,
262
  "lstrip": false,
263
  "rstrip": false,
264
+ "normalized": false,
265
+ "special": true
266
  },
267
  {
268
  "id": 52028,
 
270
  "single_word": false,
271
  "lstrip": false,
272
  "rstrip": false,
273
+ "normalized": false,
274
+ "special": true
275
  },
276
  {
277
  "id": 52029,
 
279
  "single_word": false,
280
  "lstrip": false,
281
  "rstrip": false,
282
+ "normalized": false,
283
+ "special": true
284
  },
285
  {
286
  "id": 52030,
 
288
  "single_word": false,
289
  "lstrip": false,
290
  "rstrip": false,
291
+ "normalized": false,
292
+ "special": true
293
  },
294
  {
295
  "id": 52031,
 
297
  "single_word": false,
298
  "lstrip": false,
299
  "rstrip": false,
300
+ "normalized": false,
301
+ "special": true
302
  },
303
  {
304
  "id": 52032,
 
306
  "single_word": false,
307
  "lstrip": false,
308
  "rstrip": false,
309
+ "normalized": false,
310
+ "special": true
311
  },
312
  {
313
  "id": 52033,
 
315
  "single_word": false,
316
  "lstrip": false,
317
  "rstrip": false,
318
+ "normalized": false,
319
+ "special": true
320
  },
321
  {
322
  "id": 52034,
 
324
  "single_word": false,
325
  "lstrip": false,
326
  "rstrip": false,
327
+ "normalized": false,
328
+ "special": true
329
  },
330
  {
331
  "id": 52035,
 
333
  "single_word": false,
334
  "lstrip": false,
335
  "rstrip": false,
336
+ "normalized": false,
337
+ "special": true
338
  },
339
  {
340
  "id": 52036,
 
342
  "single_word": false,
343
  "lstrip": false,
344
  "rstrip": false,
345
+ "normalized": false,
346
+ "special": true
347
  },
348
  {
349
  "id": 52037,
 
351
  "single_word": false,
352
  "lstrip": false,
353
  "rstrip": false,
354
+ "normalized": false,
355
+ "special": true
356
  },
357
  {
358
  "id": 52038,
 
360
  "single_word": false,
361
  "lstrip": false,
362
  "rstrip": false,
363
+ "normalized": false,
364
+ "special": true
365
  },
366
  {
367
  "id": 52039,
 
369
  "single_word": false,
370
  "lstrip": false,
371
  "rstrip": false,
372
+ "normalized": false,
373
+ "special": true
374
  },
375
  {
376
  "id": 52040,
 
378
  "single_word": false,
379
  "lstrip": false,
380
  "rstrip": false,
381
+ "normalized": false,
382
+ "special": true
383
  },
384
  {
385
  "id": 52041,
 
387
  "single_word": false,
388
  "lstrip": false,
389
  "rstrip": false,
390
+ "normalized": false,
391
+ "special": true
392
  },
393
  {
394
  "id": 52042,
 
396
  "single_word": false,
397
  "lstrip": false,
398
  "rstrip": false,
399
+ "normalized": false,
400
+ "special": true
401
  },
402
  {
403
  "id": 52043,
 
405
  "single_word": false,
406
  "lstrip": false,
407
  "rstrip": false,
408
+ "normalized": false,
409
+ "special": true
410
  },
411
  {
412
  "id": 52044,
 
414
  "single_word": false,
415
  "lstrip": false,
416
  "rstrip": false,
417
+ "normalized": false,
418
+ "special": true
419
  },
420
  {
421
  "id": 52045,
 
423
  "single_word": false,
424
  "lstrip": false,
425
  "rstrip": false,
426
+ "normalized": false,
427
+ "special": true
428
  },
429
  {
430
  "id": 52046,
 
432
  "single_word": false,
433
  "lstrip": false,
434
  "rstrip": false,
435
+ "normalized": false,
436
+ "special": true
437
  },
438
  {
439
  "id": 52047,
 
441
  "single_word": false,
442
  "lstrip": false,
443
  "rstrip": false,
444
+ "normalized": false,
445
+ "special": true
446
  },
447
  {
448
  "id": 52048,
 
450
  "single_word": false,
451
  "lstrip": false,
452
  "rstrip": false,
453
+ "normalized": false,
454
+ "special": true
455
  },
456
  {
457
  "id": 52049,
 
459
  "single_word": false,
460
  "lstrip": false,
461
  "rstrip": false,
462
+ "normalized": false,
463
+ "special": true
464
  },
465
  {
466
  "id": 52050,
 
468
  "single_word": false,
469
  "lstrip": false,
470
  "rstrip": false,
471
+ "normalized": false,
472
+ "special": true
473
  },
474
  {
475
  "id": 52051,
 
477
  "single_word": false,
478
  "lstrip": false,
479
  "rstrip": false,
480
+ "normalized": false,
481
+ "special": true
482
  },
483
  {
484
  "id": 52052,
 
486
  "single_word": false,
487
  "lstrip": false,
488
  "rstrip": false,
489
+ "normalized": false,
490
+ "special": true
491
  },
492
  {
493
  "id": 52053,
 
495
  "single_word": false,
496
  "lstrip": false,
497
  "rstrip": false,
498
+ "normalized": false,
499
+ "special": true
500
  },
501
  {
502
  "id": 52054,
 
504
  "single_word": false,
505
  "lstrip": false,
506
  "rstrip": false,
507
+ "normalized": false,
508
+ "special": true
509
  },
510
  {
511
  "id": 52055,
 
513
  "single_word": false,
514
  "lstrip": false,
515
  "rstrip": false,
516
+ "normalized": false,
517
+ "special": true
518
  },
519
  {
520
  "id": 52056,
 
522
  "single_word": false,
523
  "lstrip": false,
524
  "rstrip": false,
525
+ "normalized": false,
526
+ "special": true
527
  },
528
  {
529
  "id": 52057,
 
531
  "single_word": false,
532
  "lstrip": false,
533
  "rstrip": false,
534
+ "normalized": false,
535
+ "special": true
536
  },
537
  {
538
  "id": 52058,
 
540
  "single_word": false,
541
  "lstrip": false,
542
  "rstrip": false,
543
+ "normalized": false,
544
+ "special": true
545
  },
546
  {
547
  "id": 52059,
 
549
  "single_word": false,
550
  "lstrip": false,
551
  "rstrip": false,
552
+ "normalized": false,
553
+ "special": true
554
  },
555
  {
556
  "id": 52060,
 
558
  "single_word": false,
559
  "lstrip": false,
560
  "rstrip": false,
561
+ "normalized": false,
562
+ "special": true
563
  },
564
  {
565
  "id": 52061,
 
567
  "single_word": false,
568
  "lstrip": false,
569
  "rstrip": false,
570
+ "normalized": false,
571
+ "special": true
572
  },
573
  {
574
  "id": 52062,
 
576
  "single_word": false,
577
  "lstrip": false,
578
  "rstrip": false,
579
+ "normalized": false,
580
+ "special": true
581
  },
582
  {
583
  "id": 52063,
 
585
  "single_word": false,
586
  "lstrip": false,
587
  "rstrip": false,
588
+ "normalized": false,
589
+ "special": true
590
  },
591
  {
592
  "id": 52064,
 
594
  "single_word": false,
595
  "lstrip": false,
596
  "rstrip": false,
597
+ "normalized": false,
598
+ "special": true
599
  },
600
  {
601
  "id": 52065,
 
603
  "single_word": false,
604
  "lstrip": false,
605
  "rstrip": false,
606
+ "normalized": false,
607
+ "special": true
608
  },
609
  {
610
  "id": 52066,
 
612
  "single_word": false,
613
  "lstrip": false,
614
  "rstrip": false,
615
+ "normalized": false,
616
+ "special": true
617
  },
618
  {
619
  "id": 52067,
 
621
  "single_word": false,
622
  "lstrip": false,
623
  "rstrip": false,
624
+ "normalized": false,
625
+ "special": true
626
  },
627
  {
628
  "id": 52068,
 
630
  "single_word": false,
631
  "lstrip": false,
632
  "rstrip": false,
633
+ "normalized": false,
634
+ "special": true
635
  },
636
  {
637
  "id": 52069,
 
639
  "single_word": false,
640
  "lstrip": false,
641
  "rstrip": false,
642
+ "normalized": false,
643
+ "special": true
644
  },
645
  {
646
  "id": 52070,
 
648
  "single_word": false,
649
  "lstrip": false,
650
  "rstrip": false,
651
+ "normalized": false,
652
+ "special": true
653
  },
654
  {
655
  "id": 52071,
 
657
  "single_word": false,
658
  "lstrip": false,
659
  "rstrip": false,
660
+ "normalized": false,
661
+ "special": true
662
  },
663
  {
664
  "id": 52072,
 
666
  "single_word": false,
667
  "lstrip": false,
668
  "rstrip": false,
669
+ "normalized": false,
670
+ "special": true
671
  },
672
  {
673
  "id": 52073,
 
675
  "single_word": false,
676
  "lstrip": false,
677
  "rstrip": false,
678
+ "normalized": false,
679
+ "special": true
680
  },
681
  {
682
  "id": 52074,
 
684
  "single_word": false,
685
  "lstrip": false,
686
  "rstrip": false,
687
+ "normalized": false,
688
+ "special": true
689
  },
690
  {
691
  "id": 52075,
 
693
  "single_word": false,
694
  "lstrip": false,
695
  "rstrip": false,
696
+ "normalized": false,
697
+ "special": true
698
  },
699
  {
700
  "id": 52076,
 
702
  "single_word": false,
703
  "lstrip": false,
704
  "rstrip": false,
705
+ "normalized": false,
706
+ "special": true
707
  },
708
  {
709
  "id": 52077,
 
711
  "single_word": false,
712
  "lstrip": false,
713
  "rstrip": false,
714
+ "normalized": false,
715
+ "special": true
716
  },
717
  {
718
  "id": 52078,
 
720
  "single_word": false,
721
  "lstrip": false,
722
  "rstrip": false,
723
+ "normalized": false,
724
+ "special": true
725
  },
726
  {
727
  "id": 52079,
 
729
  "single_word": false,
730
  "lstrip": false,
731
  "rstrip": false,
732
+ "normalized": false,
733
+ "special": true
734
  },
735
  {
736
  "id": 52080,
 
738
  "single_word": false,
739
  "lstrip": false,
740
  "rstrip": false,
741
+ "normalized": false,
742
+ "special": true
743
  },
744
  {
745
  "id": 52081,
 
747
  "single_word": false,
748
  "lstrip": false,
749
  "rstrip": false,
750
+ "normalized": false,
751
+ "special": true
752
  },
753
  {
754
  "id": 52082,
 
756
  "single_word": false,
757
  "lstrip": false,
758
  "rstrip": false,
759
+ "normalized": false,
760
+ "special": true
761
  },
762
  {
763
  "id": 52083,
 
765
  "single_word": false,
766
  "lstrip": false,
767
  "rstrip": false,
768
+ "normalized": false,
769
+ "special": true
770
  },
771
  {
772
  "id": 52084,
 
774
  "single_word": false,
775
  "lstrip": false,
776
  "rstrip": false,
777
+ "normalized": false,
778
+ "special": true
779
  },
780
  {
781
  "id": 52085,
 
783
  "single_word": false,
784
  "lstrip": false,
785
  "rstrip": false,
786
+ "normalized": false,
787
+ "special": true
788
  },
789
  {
790
  "id": 52086,
 
792
  "single_word": false,
793
  "lstrip": false,
794
  "rstrip": false,
795
+ "normalized": false,
796
+ "special": true
797
  },
798
  {
799
  "id": 52087,
 
801
  "single_word": false,
802
  "lstrip": false,
803
  "rstrip": false,
804
+ "normalized": false,
805
+ "special": true
806
  },
807
  {
808
  "id": 52088,
 
810
  "single_word": false,
811
  "lstrip": false,
812
  "rstrip": false,
813
+ "normalized": false,
814
+ "special": true
815
  },
816
  {
817
  "id": 52089,
 
819
  "single_word": false,
820
  "lstrip": false,
821
  "rstrip": false,
822
+ "normalized": false,
823
+ "special": true
824
  },
825
  {
826
  "id": 52090,
 
828
  "single_word": false,
829
  "lstrip": false,
830
  "rstrip": false,
831
+ "normalized": false,
832
+ "special": true
833
  },
834
  {
835
  "id": 52091,
 
837
  "single_word": false,
838
  "lstrip": false,
839
  "rstrip": false,
840
+ "normalized": false,
841
+ "special": true
842
  },
843
  {
844
  "id": 52092,
 
846
  "single_word": false,
847
  "lstrip": false,
848
  "rstrip": false,
849
+ "normalized": false,
850
+ "special": true
851
  },
852
  {
853
  "id": 52093,
 
855
  "single_word": false,
856
  "lstrip": false,
857
  "rstrip": false,
858
+ "normalized": false,
859
+ "special": true
860
  },
861
  {
862
  "id": 52094,
 
864
  "single_word": false,
865
  "lstrip": false,
866
  "rstrip": false,
867
+ "normalized": false,
868
+ "special": true
869
  },
870
  {
871
  "id": 52095,
 
873
  "single_word": false,
874
  "lstrip": false,
875
  "rstrip": false,
876
+ "normalized": false,
877
+ "special": true
878
  },
879
  {
880
  "id": 52096,
 
882
  "single_word": false,
883
  "lstrip": false,
884
  "rstrip": false,
885
+ "normalized": false,
886
+ "special": true
887
  },
888
  {
889
  "id": 52097,
 
891
  "single_word": false,
892
  "lstrip": false,
893
  "rstrip": false,
894
+ "normalized": false,
895
+ "special": true
896
  },
897
  {
898
  "id": 52098,
 
900
  "single_word": false,
901
  "lstrip": false,
902
  "rstrip": false,
903
+ "normalized": false,
904
+ "special": true
905
  },
906
  {
907
  "id": 52099,
 
909
  "single_word": false,
910
  "lstrip": false,
911
  "rstrip": false,
912
+ "normalized": false,
913
+ "special": true
914
  },
915
  {
916
  "id": 52100,
 
918
  "single_word": false,
919
  "lstrip": false,
920
  "rstrip": false,
921
+ "normalized": false,
922
+ "special": true
923
  },
924
  {
925
  "id": 52101,
 
927
  "single_word": false,
928
  "lstrip": false,
929
  "rstrip": false,
930
+ "normalized": false,
931
+ "special": true
932
  },
933
  {
934
  "id": 52102,
 
936
  "single_word": false,
937
  "lstrip": false,
938
  "rstrip": false,
939
+ "normalized": false,
940
+ "special": true
941
  },
942
  {
943
  "id": 52103,
 
945
  "single_word": false,
946
  "lstrip": false,
947
  "rstrip": false,
948
+ "normalized": false,
949
+ "special": true
950
  }
951
  ],
952
  "normalizer": null,
tokenizer_config.json CHANGED
@@ -3,7 +3,7 @@
3
  "bos_token": "<|endoftext|>",
4
  "eos_token": "<|endoftext|>",
5
  "model_max_length": 1024,
6
- "name_or_path": "MarkGG/Romance-cleaned-1",
7
  "special_tokens_map_file": null,
8
  "tokenizer_class": "GPT2Tokenizer",
9
  "unk_token": "<|endoftext|>"
 
3
  "bos_token": "<|endoftext|>",
4
  "eos_token": "<|endoftext|>",
5
  "model_max_length": 1024,
6
+ "name_or_path": "gpt2",
7
  "special_tokens_map_file": null,
8
  "tokenizer_class": "GPT2Tokenizer",
9
  "unk_token": "<|endoftext|>"