Hugging Face's logo

Humair332
/
gpt-style-50k-tokenizer

Model card Files
xet
 Community
4

feat: inject 80 sound/paralinguistic tags as single added_tokens

#2
by zuhri025 - opened
base: refs/heads/main
←
from: refs/pr/2
Discussion Files changed
+1525
-3
Files changed (2) hide show
  1. tokenizer.json +1443 -1
  2. tokenizer_config.json +82 -2
tokenizer.json CHANGED
@@ -830,6 +830,726 @@
830
  "rstrip": false,
831
  "normalized": false,
832
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
833
  }
834
  ],
835
  "normalizer": {
@@ -50882,7 +51602,87 @@
50882
  "ĠBened": 49996,
50883
  "ĠÙĤصÛĮدÛĴ": 49997,
50884
  "Complete": 49998,
50885
- "اÙĦÙģضÙĦ": 49999
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50886
  },
50887
  "merges": [
50888
  [
@@ -249494,5 +250294,647 @@
249494
  "ضÙĦ"
249495
  ]
249496
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249497
  }
249498
  }
 
830
  "rstrip": false,
831
  "normalized": false,
832
  "special": true
833
+ },
834
+ {
835
+ "id": 50000,
836
+ "content": "[LAUGHS]",
837
+ "single_word": false,
838
+ "lstrip": false,
839
+ "rstrip": false,
840
+ "normalized": false,
841
+ "special": true
842
+ },
843
+ {
844
+ "id": 50001,
845
+ "content": "[SIGHS]",
846
+ "single_word": false,
847
+ "lstrip": false,
848
+ "rstrip": false,
849
+ "normalized": false,
850
+ "special": true
851
+ },
852
+ {
853
+ "id": 50002,
854
+ "content": "[GASPS]",
855
+ "single_word": false,
856
+ "lstrip": false,
857
+ "rstrip": false,
858
+ "normalized": false,
859
+ "special": true
860
+ },
861
+ {
862
+ "id": 50003,
863
+ "content": "[CLEARS_THROAT]",
864
+ "single_word": false,
865
+ "lstrip": false,
866
+ "rstrip": false,
867
+ "normalized": false,
868
+ "special": true
869
+ },
870
+ {
871
+ "id": 50004,
872
+ "content": "[MUSIC]",
873
+ "single_word": false,
874
+ "lstrip": false,
875
+ "rstrip": false,
876
+ "normalized": false,
877
+ "special": true
878
+ },
879
+ {
880
+ "id": 50005,
881
+ "content": "[BEEP]",
882
+ "single_word": false,
883
+ "lstrip": false,
884
+ "rstrip": false,
885
+ "normalized": false,
886
+ "special": true
887
+ },
888
+ {
889
+ "id": 50006,
890
+ "content": "[APPLAUSE]",
891
+ "single_word": false,
892
+ "lstrip": false,
893
+ "rstrip": false,
894
+ "normalized": false,
895
+ "special": true
896
+ },
897
+ {
898
+ "id": 50007,
899
+ "content": "[COUGHS]",
900
+ "single_word": false,
901
+ "lstrip": false,
902
+ "rstrip": false,
903
+ "normalized": false,
904
+ "special": true
905
+ },
906
+ {
907
+ "id": 50008,
908
+ "content": "[MUMBLES]",
909
+ "single_word": false,
910
+ "lstrip": false,
911
+ "rstrip": false,
912
+ "normalized": false,
913
+ "special": true
914
+ },
915
+ {
916
+ "id": 50009,
917
+ "content": "[WHISTLES]",
918
+ "single_word": false,
919
+ "lstrip": false,
920
+ "rstrip": false,
921
+ "normalized": false,
922
+ "special": true
923
+ },
924
+ {
925
+ "id": 50010,
926
+ "content": "[SNIFFS]",
927
+ "single_word": false,
928
+ "lstrip": false,
929
+ "rstrip": false,
930
+ "normalized": false,
931
+ "special": true
932
+ },
933
+ {
934
+ "id": 50011,
935
+ "content": "[SINGS]",
936
+ "single_word": false,
937
+ "lstrip": false,
938
+ "rstrip": false,
939
+ "normalized": false,
940
+ "special": true
941
+ },
942
+ {
943
+ "id": 50012,
944
+ "content": "[GROANS]",
945
+ "single_word": false,
946
+ "lstrip": false,
947
+ "rstrip": false,
948
+ "normalized": false,
949
+ "special": true
950
+ },
951
+ {
952
+ "id": 50013,
953
+ "content": "[SCREAMS]",
954
+ "single_word": false,
955
+ "lstrip": false,
956
+ "rstrip": false,
957
+ "normalized": false,
958
+ "special": true
959
+ },
960
+ {
961
+ "id": 50014,
962
+ "content": "[SNEEZES]",
963
+ "single_word": false,
964
+ "lstrip": false,
965
+ "rstrip": false,
966
+ "normalized": false,
967
+ "special": true
968
+ },
969
+ {
970
+ "id": 50015,
971
+ "content": "[EXHALES]",
972
+ "single_word": false,
973
+ "lstrip": false,
974
+ "rstrip": false,
975
+ "normalized": false,
976
+ "special": true
977
+ },
978
+ {
979
+ "id": 50016,
980
+ "content": "[INHALES]",
981
+ "single_word": false,
982
+ "lstrip": false,
983
+ "rstrip": false,
984
+ "normalized": false,
985
+ "special": true
986
+ },
987
+ {
988
+ "id": 50017,
989
+ "content": "[BREATH]",
990
+ "single_word": false,
991
+ "lstrip": false,
992
+ "rstrip": false,
993
+ "normalized": false,
994
+ "special": true
995
+ },
996
+ {
997
+ "id": 50018,
998
+ "content": "[DEEP_BREATH]",
999
+ "single_word": false,
1000
+ "lstrip": false,
1001
+ "rstrip": false,
1002
+ "normalized": false,
1003
+ "special": true
1004
+ },
1005
+ {
1006
+ "id": 50019,
1007
+ "content": "[SHARP_INHALE]",
1008
+ "single_word": false,
1009
+ "lstrip": false,
1010
+ "rstrip": false,
1011
+ "normalized": false,
1012
+ "special": true
1013
+ },
1014
+ {
1015
+ "id": 50020,
1016
+ "content": "[SHARP_BREATH]",
1017
+ "single_word": false,
1018
+ "lstrip": false,
1019
+ "rstrip": false,
1020
+ "normalized": false,
1021
+ "special": true
1022
+ },
1023
+ {
1024
+ "id": 50021,
1025
+ "content": "[SHAKY_BREATH]",
1026
+ "single_word": false,
1027
+ "lstrip": false,
1028
+ "rstrip": false,
1029
+ "normalized": false,
1030
+ "special": true
1031
+ },
1032
+ {
1033
+ "id": 50022,
1034
+ "content": "[BREATHY_EXHALE]",
1035
+ "single_word": false,
1036
+ "lstrip": false,
1037
+ "rstrip": false,
1038
+ "normalized": false,
1039
+ "special": true
1040
+ },
1041
+ {
1042
+ "id": 50023,
1043
+ "content": "[STRAINED_BREATH]",
1044
+ "single_word": false,
1045
+ "lstrip": false,
1046
+ "rstrip": false,
1047
+ "normalized": false,
1048
+ "special": true
1049
+ },
1050
+ {
1051
+ "id": 50024,
1052
+ "content": "[QUICK_BREATH]",
1053
+ "single_word": false,
1054
+ "lstrip": false,
1055
+ "rstrip": false,
1056
+ "normalized": false,
1057
+ "special": true
1058
+ },
1059
+ {
1060
+ "id": 50025,
1061
+ "content": "[PANTING]",
1062
+ "single_word": false,
1063
+ "lstrip": false,
1064
+ "rstrip": false,
1065
+ "normalized": false,
1066
+ "special": true
1067
+ },
1068
+ {
1069
+ "id": 50026,
1070
+ "content": "[BREATHING]",
1071
+ "single_word": false,
1072
+ "lstrip": false,
1073
+ "rstrip": false,
1074
+ "normalized": false,
1075
+ "special": true
1076
+ },
1077
+ {
1078
+ "id": 50027,
1079
+ "content": "[HEAVY_BREATHING]",
1080
+ "single_word": false,
1081
+ "lstrip": false,
1082
+ "rstrip": false,
1083
+ "normalized": false,
1084
+ "special": true
1085
+ },
1086
+ {
1087
+ "id": 50028,
1088
+ "content": "[NERVOUS_EXHALE]",
1089
+ "single_word": false,
1090
+ "lstrip": false,
1091
+ "rstrip": false,
1092
+ "normalized": false,
1093
+ "special": true
1094
+ },
1095
+ {
1096
+ "id": 50029,
1097
+ "content": "[PAINED_BREATH]",
1098
+ "single_word": false,
1099
+ "lstrip": false,
1100
+ "rstrip": false,
1101
+ "normalized": false,
1102
+ "special": true
1103
+ },
1104
+ {
1105
+ "id": 50030,
1106
+ "content": "[GIGGLES]",
1107
+ "single_word": false,
1108
+ "lstrip": false,
1109
+ "rstrip": false,
1110
+ "normalized": false,
1111
+ "special": true
1112
+ },
1113
+ {
1114
+ "id": 50031,
1115
+ "content": "[CHUCKLES]",
1116
+ "single_word": false,
1117
+ "lstrip": false,
1118
+ "rstrip": false,
1119
+ "normalized": false,
1120
+ "special": true
1121
+ },
1122
+ {
1123
+ "id": 50032,
1124
+ "content": "[SOFT_CHUCKLE]",
1125
+ "single_word": false,
1126
+ "lstrip": false,
1127
+ "rstrip": false,
1128
+ "normalized": false,
1129
+ "special": true
1130
+ },
1131
+ {
1132
+ "id": 50033,
1133
+ "content": "[SOFT_LAUGH]",
1134
+ "single_word": false,
1135
+ "lstrip": false,
1136
+ "rstrip": false,
1137
+ "normalized": false,
1138
+ "special": true
1139
+ },
1140
+ {
1141
+ "id": 50034,
1142
+ "content": "[NERVOUS_LAUGH]",
1143
+ "single_word": false,
1144
+ "lstrip": false,
1145
+ "rstrip": false,
1146
+ "normalized": false,
1147
+ "special": true
1148
+ },
1149
+ {
1150
+ "id": 50035,
1151
+ "content": "[SOBS]",
1152
+ "single_word": false,
1153
+ "lstrip": false,
1154
+ "rstrip": false,
1155
+ "normalized": false,
1156
+ "special": true
1157
+ },
1158
+ {
1159
+ "id": 50036,
1160
+ "content": "[SOBBING]",
1161
+ "single_word": false,
1162
+ "lstrip": false,
1163
+ "rstrip": false,
1164
+ "normalized": false,
1165
+ "special": true
1166
+ },
1167
+ {
1168
+ "id": 50037,
1169
+ "content": "[CHOKED_SOB]",
1170
+ "single_word": false,
1171
+ "lstrip": false,
1172
+ "rstrip": false,
1173
+ "normalized": false,
1174
+ "special": true
1175
+ },
1176
+ {
1177
+ "id": 50038,
1178
+ "content": "[CRYING]",
1179
+ "single_word": false,
1180
+ "lstrip": false,
1181
+ "rstrip": false,
1182
+ "normalized": false,
1183
+ "special": true
1184
+ },
1185
+ {
1186
+ "id": 50039,
1187
+ "content": "[WHIMPERS]",
1188
+ "single_word": false,
1189
+ "lstrip": false,
1190
+ "rstrip": false,
1191
+ "normalized": false,
1192
+ "special": true
1193
+ },
1194
+ {
1195
+ "id": 50040,
1196
+ "content": "[SOFT_WHIMPER]",
1197
+ "single_word": false,
1198
+ "lstrip": false,
1199
+ "rstrip": false,
1200
+ "normalized": false,
1201
+ "special": true
1202
+ },
1203
+ {
1204
+ "id": 50041,
1205
+ "content": "[PAINED_CRY]",
1206
+ "single_word": false,
1207
+ "lstrip": false,
1208
+ "rstrip": false,
1209
+ "normalized": false,
1210
+ "special": true
1211
+ },
1212
+ {
1213
+ "id": 50042,
1214
+ "content": "[PAINED_GRUNT]",
1215
+ "single_word": false,
1216
+ "lstrip": false,
1217
+ "rstrip": false,
1218
+ "normalized": false,
1219
+ "special": true
1220
+ },
1221
+ {
1222
+ "id": 50043,
1223
+ "content": "[PAINED_SOUND]",
1224
+ "single_word": false,
1225
+ "lstrip": false,
1226
+ "rstrip": false,
1227
+ "normalized": false,
1228
+ "special": true
1229
+ },
1230
+ {
1231
+ "id": 50044,
1232
+ "content": "[CHOKES]",
1233
+ "single_word": false,
1234
+ "lstrip": false,
1235
+ "rstrip": false,
1236
+ "normalized": false,
1237
+ "special": true
1238
+ },
1239
+ {
1240
+ "id": 50045,
1241
+ "content": "[HUMS]",
1242
+ "single_word": false,
1243
+ "lstrip": false,
1244
+ "rstrip": false,
1245
+ "normalized": false,
1246
+ "special": true
1247
+ },
1248
+ {
1249
+ "id": 50046,
1250
+ "content": "[HUMMING]",
1251
+ "single_word": false,
1252
+ "lstrip": false,
1253
+ "rstrip": false,
1254
+ "normalized": false,
1255
+ "special": true
1256
+ },
1257
+ {
1258
+ "id": 50047,
1259
+ "content": "[SOFT_HUM]",
1260
+ "single_word": false,
1261
+ "lstrip": false,
1262
+ "rstrip": false,
1263
+ "normalized": false,
1264
+ "special": true
1265
+ },
1266
+ {
1267
+ "id": 50048,
1268
+ "content": "[PLEASED_HUM]",
1269
+ "single_word": false,
1270
+ "lstrip": false,
1271
+ "rstrip": false,
1272
+ "normalized": false,
1273
+ "special": true
1274
+ },
1275
+ {
1276
+ "id": 50049,
1277
+ "content": "[CONTENTED_HUM]",
1278
+ "single_word": false,
1279
+ "lstrip": false,
1280
+ "rstrip": false,
1281
+ "normalized": false,
1282
+ "special": true
1283
+ },
1284
+ {
1285
+ "id": 50050,
1286
+ "content": "[MURMURS]",
1287
+ "single_word": false,
1288
+ "lstrip": false,
1289
+ "rstrip": false,
1290
+ "normalized": false,
1291
+ "special": true
1292
+ },
1293
+ {
1294
+ "id": 50051,
1295
+ "content": "Mmm",
1296
+ "single_word": false,
1297
+ "lstrip": false,
1298
+ "rstrip": false,
1299
+ "normalized": false,
1300
+ "special": true
1301
+ },
1302
+ {
1303
+ "id": 50052,
1304
+ "content": "Hmm",
1305
+ "single_word": false,
1306
+ "lstrip": false,
1307
+ "rstrip": false,
1308
+ "normalized": false,
1309
+ "special": true
1310
+ },
1311
+ {
1312
+ "id": 50053,
1313
+ "content": "[HMM]",
1314
+ "single_word": false,
1315
+ "lstrip": false,
1316
+ "rstrip": false,
1317
+ "normalized": false,
1318
+ "special": true
1319
+ },
1320
+ {
1321
+ "id": 50054,
1322
+ "content": "[AH]",
1323
+ "single_word": false,
1324
+ "lstrip": false,
1325
+ "rstrip": false,
1326
+ "normalized": false,
1327
+ "special": true
1328
+ },
1329
+ {
1330
+ "id": 50055,
1331
+ "content": "[MOUTH_SOUND]",
1332
+ "single_word": false,
1333
+ "lstrip": false,
1334
+ "rstrip": false,
1335
+ "normalized": false,
1336
+ "special": true
1337
+ },
1338
+ {
1339
+ "id": 50056,
1340
+ "content": "[MOUTH_CLICK]",
1341
+ "single_word": false,
1342
+ "lstrip": false,
1343
+ "rstrip": false,
1344
+ "normalized": false,
1345
+ "special": true
1346
+ },
1347
+ {
1348
+ "id": 50057,
1349
+ "content": "[TONGUE_CLICK]",
1350
+ "single_word": false,
1351
+ "lstrip": false,
1352
+ "rstrip": false,
1353
+ "normalized": false,
1354
+ "special": true
1355
+ },
1356
+ {
1357
+ "id": 50058,
1358
+ "content": "[LIP_SMACK]",
1359
+ "single_word": false,
1360
+ "lstrip": false,
1361
+ "rstrip": false,
1362
+ "normalized": false,
1363
+ "special": true
1364
+ },
1365
+ {
1366
+ "id": 50059,
1367
+ "content": "[SMACKS]",
1368
+ "single_word": false,
1369
+ "lstrip": false,
1370
+ "rstrip": false,
1371
+ "normalized": false,
1372
+ "special": true
1373
+ },
1374
+ {
1375
+ "id": 50060,
1376
+ "content": "[TSK]",
1377
+ "single_word": false,
1378
+ "lstrip": false,
1379
+ "rstrip": false,
1380
+ "normalized": false,
1381
+ "special": true
1382
+ },
1383
+ {
1384
+ "id": 50061,
1385
+ "content": "[CLICKS]",
1386
+ "single_word": false,
1387
+ "lstrip": false,
1388
+ "rstrip": false,
1389
+ "normalized": false,
1390
+ "special": true
1391
+ },
1392
+ {
1393
+ "id": 50062,
1394
+ "content": "[SLURPS]",
1395
+ "single_word": false,
1396
+ "lstrip": false,
1397
+ "rstrip": false,
1398
+ "normalized": false,
1399
+ "special": true
1400
+ },
1401
+ {
1402
+ "id": 50063,
1403
+ "content": "[GULPS]",
1404
+ "single_word": false,
1405
+ "lstrip": false,
1406
+ "rstrip": false,
1407
+ "normalized": false,
1408
+ "special": true
1409
+ },
1410
+ {
1411
+ "id": 50064,
1412
+ "content": "[SWALLOWS]",
1413
+ "single_word": false,
1414
+ "lstrip": false,
1415
+ "rstrip": false,
1416
+ "normalized": false,
1417
+ "special": true
1418
+ },
1419
+ {
1420
+ "id": 50065,
1421
+ "content": "[YAWNS]",
1422
+ "single_word": false,
1423
+ "lstrip": false,
1424
+ "rstrip": false,
1425
+ "normalized": false,
1426
+ "special": true
1427
+ },
1428
+ {
1429
+ "id": 50066,
1430
+ "content": "[MOANS]",
1431
+ "single_word": false,
1432
+ "lstrip": false,
1433
+ "rstrip": false,
1434
+ "normalized": false,
1435
+ "special": true
1436
+ },
1437
+ {
1438
+ "id": 50067,
1439
+ "content": "[GRUNTS]",
1440
+ "single_word": false,
1441
+ "lstrip": false,
1442
+ "rstrip": false,
1443
+ "normalized": false,
1444
+ "special": true
1445
+ },
1446
+ {
1447
+ "id": 50068,
1448
+ "content": "[GROWLS]",
1449
+ "single_word": false,
1450
+ "lstrip": false,
1451
+ "rstrip": false,
1452
+ "normalized": false,
1453
+ "special": true
1454
+ },
1455
+ {
1456
+ "id": 50069,
1457
+ "content": "[HISSES]",
1458
+ "single_word": false,
1459
+ "lstrip": false,
1460
+ "rstrip": false,
1461
+ "normalized": false,
1462
+ "special": true
1463
+ },
1464
+ {
1465
+ "id": 50070,
1466
+ "content": "[SCOFFS]",
1467
+ "single_word": false,
1468
+ "lstrip": false,
1469
+ "rstrip": false,
1470
+ "normalized": false,
1471
+ "special": true
1472
+ },
1473
+ {
1474
+ "id": 50071,
1475
+ "content": "[HMPH]",
1476
+ "single_word": false,
1477
+ "lstrip": false,
1478
+ "rstrip": false,
1479
+ "normalized": false,
1480
+ "special": true
1481
+ },
1482
+ {
1483
+ "id": 50072,
1484
+ "content": "[HUFFS]",
1485
+ "single_word": false,
1486
+ "lstrip": false,
1487
+ "rstrip": false,
1488
+ "normalized": false,
1489
+ "special": true
1490
+ },
1491
+ {
1492
+ "id": 50073,
1493
+ "content": "[WHISPERS]",
1494
+ "single_word": false,
1495
+ "lstrip": false,
1496
+ "rstrip": false,
1497
+ "normalized": false,
1498
+ "special": true
1499
+ },
1500
+ {
1501
+ "id": 50074,
1502
+ "content": "[PURRS]",
1503
+ "single_word": false,
1504
+ "lstrip": false,
1505
+ "rstrip": false,
1506
+ "normalized": false,
1507
+ "special": true
1508
+ },
1509
+ {
1510
+ "id": 50075,
1511
+ "content": "[STUTTERS]",
1512
+ "single_word": false,
1513
+ "lstrip": false,
1514
+ "rstrip": false,
1515
+ "normalized": false,
1516
+ "special": true
1517
+ },
1518
+ {
1519
+ "id": 50076,
1520
+ "content": "[HICCUPS]",
1521
+ "single_word": false,
1522
+ "lstrip": false,
1523
+ "rstrip": false,
1524
+ "normalized": false,
1525
+ "special": true
1526
+ },
1527
+ {
1528
+ "id": 50077,
1529
+ "content": "[SHIVERS]",
1530
+ "single_word": false,
1531
+ "lstrip": false,
1532
+ "rstrip": false,
1533
+ "normalized": false,
1534
+ "special": true
1535
+ },
1536
+ {
1537
+ "id": 50078,
1538
+ "content": "[YELPS]",
1539
+ "single_word": false,
1540
+ "lstrip": false,
1541
+ "rstrip": false,
1542
+ "normalized": false,
1543
+ "special": true
1544
+ },
1545
+ {
1546
+ "id": 50079,
1547
+ "content": "[ROARS]",
1548
+ "single_word": false,
1549
+ "lstrip": false,
1550
+ "rstrip": false,
1551
+ "normalized": false,
1552
+ "special": true
1553
  }
1554
  ],
1555
  "normalizer": {
 
51602
  "ĠBened": 49996,
51603
  "ĠÙĤصÛĮدÛĴ": 49997,
51604
  "Complete": 49998,
51605
+ "اÙĦÙģضÙĦ": 49999,
51606
+ "[LAUGHS]": 50000,
51607
+ "[SIGHS]": 50001,
51608
+ "[GASPS]": 50002,
51609
+ "[CLEARS_THROAT]": 50003,
51610
+ "[MUSIC]": 50004,
51611
+ "[BEEP]": 50005,
51612
+ "[APPLAUSE]": 50006,
51613
+ "[COUGHS]": 50007,
51614
+ "[MUMBLES]": 50008,
51615
+ "[WHISTLES]": 50009,
51616
+ "[SNIFFS]": 50010,
51617
+ "[SINGS]": 50011,
51618
+ "[GROANS]": 50012,
51619
+ "[SCREAMS]": 50013,
51620
+ "[SNEEZES]": 50014,
51621
+ "[EXHALES]": 50015,
51622
+ "[INHALES]": 50016,
51623
+ "[BREATH]": 50017,
51624
+ "[DEEP_BREATH]": 50018,
51625
+ "[SHARP_INHALE]": 50019,
51626
+ "[SHARP_BREATH]": 50020,
51627
+ "[SHAKY_BREATH]": 50021,
51628
+ "[BREATHY_EXHALE]": 50022,
51629
+ "[STRAINED_BREATH]": 50023,
51630
+ "[QUICK_BREATH]": 50024,
51631
+ "[PANTING]": 50025,
51632
+ "[BREATHING]": 50026,
51633
+ "[HEAVY_BREATHING]": 50027,
51634
+ "[NERVOUS_EXHALE]": 50028,
51635
+ "[PAINED_BREATH]": 50029,
51636
+ "[GIGGLES]": 50030,
51637
+ "[CHUCKLES]": 50031,
51638
+ "[SOFT_CHUCKLE]": 50032,
51639
+ "[SOFT_LAUGH]": 50033,
51640
+ "[NERVOUS_LAUGH]": 50034,
51641
+ "[SOBS]": 50035,
51642
+ "[SOBBING]": 50036,
51643
+ "[CHOKED_SOB]": 50037,
51644
+ "[CRYING]": 50038,
51645
+ "[WHIMPERS]": 50039,
51646
+ "[SOFT_WHIMPER]": 50040,
51647
+ "[PAINED_CRY]": 50041,
51648
+ "[PAINED_GRUNT]": 50042,
51649
+ "[PAINED_SOUND]": 50043,
51650
+ "[CHOKES]": 50044,
51651
+ "[HUMS]": 50045,
51652
+ "[HUMMING]": 50046,
51653
+ "[SOFT_HUM]": 50047,
51654
+ "[PLEASED_HUM]": 50048,
51655
+ "[CONTENTED_HUM]": 50049,
51656
+ "[MURMURS]": 50050,
51657
+ "Mmm": 50051,
51658
+ "Hmm": 50052,
51659
+ "[HMM]": 50053,
51660
+ "[AH]": 50054,
51661
+ "[MOUTH_SOUND]": 50055,
51662
+ "[MOUTH_CLICK]": 50056,
51663
+ "[TONGUE_CLICK]": 50057,
51664
+ "[LIP_SMACK]": 50058,
51665
+ "[SMACKS]": 50059,
51666
+ "[TSK]": 50060,
51667
+ "[CLICKS]": 50061,
51668
+ "[SLURPS]": 50062,
51669
+ "[GULPS]": 50063,
51670
+ "[SWALLOWS]": 50064,
51671
+ "[YAWNS]": 50065,
51672
+ "[MOANS]": 50066,
51673
+ "[GRUNTS]": 50067,
51674
+ "[GROWLS]": 50068,
51675
+ "[HISSES]": 50069,
51676
+ "[SCOFFS]": 50070,
51677
+ "[HMPH]": 50071,
51678
+ "[HUFFS]": 50072,
51679
+ "[WHISPERS]": 50073,
51680
+ "[PURRS]": 50074,
51681
+ "[STUTTERS]": 50075,
51682
+ "[HICCUPS]": 50076,
51683
+ "[SHIVERS]": 50077,
51684
+ "[YELPS]": 50078,
51685
+ "[ROARS]": 50079
51686
  },
51687
  "merges": [
51688
  [
 
250294
  "ضÙĦ"
250295
  ]
250296
  ]
250297
+ },
250298
+ "added_tokens_decoder": {
250299
+ "50000": {
250300
+ "content": "[LAUGHS]",
250301
+ "single_word": false,
250302
+ "lstrip": false,
250303
+ "rstrip": false,
250304
+ "normalized": false,
250305
+ "special": true
250306
+ },
250307
+ "50001": {
250308
+ "content": "[SIGHS]",
250309
+ "single_word": false,
250310
+ "lstrip": false,
250311
+ "rstrip": false,
250312
+ "normalized": false,
250313
+ "special": true
250314
+ },
250315
+ "50002": {
250316
+ "content": "[GASPS]",
250317
+ "single_word": false,
250318
+ "lstrip": false,
250319
+ "rstrip": false,
250320
+ "normalized": false,
250321
+ "special": true
250322
+ },
250323
+ "50003": {
250324
+ "content": "[CLEARS_THROAT]",
250325
+ "single_word": false,
250326
+ "lstrip": false,
250327
+ "rstrip": false,
250328
+ "normalized": false,
250329
+ "special": true
250330
+ },
250331
+ "50004": {
250332
+ "content": "[MUSIC]",
250333
+ "single_word": false,
250334
+ "lstrip": false,
250335
+ "rstrip": false,
250336
+ "normalized": false,
250337
+ "special": true
250338
+ },
250339
+ "50005": {
250340
+ "content": "[BEEP]",
250341
+ "single_word": false,
250342
+ "lstrip": false,
250343
+ "rstrip": false,
250344
+ "normalized": false,
250345
+ "special": true
250346
+ },
250347
+ "50006": {
250348
+ "content": "[APPLAUSE]",
250349
+ "single_word": false,
250350
+ "lstrip": false,
250351
+ "rstrip": false,
250352
+ "normalized": false,
250353
+ "special": true
250354
+ },
250355
+ "50007": {
250356
+ "content": "[COUGHS]",
250357
+ "single_word": false,
250358
+ "lstrip": false,
250359
+ "rstrip": false,
250360
+ "normalized": false,
250361
+ "special": true
250362
+ },
250363
+ "50008": {
250364
+ "content": "[MUMBLES]",
250365
+ "single_word": false,
250366
+ "lstrip": false,
250367
+ "rstrip": false,
250368
+ "normalized": false,
250369
+ "special": true
250370
+ },
250371
+ "50009": {
250372
+ "content": "[WHISTLES]",
250373
+ "single_word": false,
250374
+ "lstrip": false,
250375
+ "rstrip": false,
250376
+ "normalized": false,
250377
+ "special": true
250378
+ },
250379
+ "50010": {
250380
+ "content": "[SNIFFS]",
250381
+ "single_word": false,
250382
+ "lstrip": false,
250383
+ "rstrip": false,
250384
+ "normalized": false,
250385
+ "special": true
250386
+ },
250387
+ "50011": {
250388
+ "content": "[SINGS]",
250389
+ "single_word": false,
250390
+ "lstrip": false,
250391
+ "rstrip": false,
250392
+ "normalized": false,
250393
+ "special": true
250394
+ },
250395
+ "50012": {
250396
+ "content": "[GROANS]",
250397
+ "single_word": false,
250398
+ "lstrip": false,
250399
+ "rstrip": false,
250400
+ "normalized": false,
250401
+ "special": true
250402
+ },
250403
+ "50013": {
250404
+ "content": "[SCREAMS]",
250405
+ "single_word": false,
250406
+ "lstrip": false,
250407
+ "rstrip": false,
250408
+ "normalized": false,
250409
+ "special": true
250410
+ },
250411
+ "50014": {
250412
+ "content": "[SNEEZES]",
250413
+ "single_word": false,
250414
+ "lstrip": false,
250415
+ "rstrip": false,
250416
+ "normalized": false,
250417
+ "special": true
250418
+ },
250419
+ "50015": {
250420
+ "content": "[EXHALES]",
250421
+ "single_word": false,
250422
+ "lstrip": false,
250423
+ "rstrip": false,
250424
+ "normalized": false,
250425
+ "special": true
250426
+ },
250427
+ "50016": {
250428
+ "content": "[INHALES]",
250429
+ "single_word": false,
250430
+ "lstrip": false,
250431
+ "rstrip": false,
250432
+ "normalized": false,
250433
+ "special": true
250434
+ },
250435
+ "50017": {
250436
+ "content": "[BREATH]",
250437
+ "single_word": false,
250438
+ "lstrip": false,
250439
+ "rstrip": false,
250440
+ "normalized": false,
250441
+ "special": true
250442
+ },
250443
+ "50018": {
250444
+ "content": "[DEEP_BREATH]",
250445
+ "single_word": false,
250446
+ "lstrip": false,
250447
+ "rstrip": false,
250448
+ "normalized": false,
250449
+ "special": true
250450
+ },
250451
+ "50019": {
250452
+ "content": "[SHARP_INHALE]",
250453
+ "single_word": false,
250454
+ "lstrip": false,
250455
+ "rstrip": false,
250456
+ "normalized": false,
250457
+ "special": true
250458
+ },
250459
+ "50020": {
250460
+ "content": "[SHARP_BREATH]",
250461
+ "single_word": false,
250462
+ "lstrip": false,
250463
+ "rstrip": false,
250464
+ "normalized": false,
250465
+ "special": true
250466
+ },
250467
+ "50021": {
250468
+ "content": "[SHAKY_BREATH]",
250469
+ "single_word": false,
250470
+ "lstrip": false,
250471
+ "rstrip": false,
250472
+ "normalized": false,
250473
+ "special": true
250474
+ },
250475
+ "50022": {
250476
+ "content": "[BREATHY_EXHALE]",
250477
+ "single_word": false,
250478
+ "lstrip": false,
250479
+ "rstrip": false,
250480
+ "normalized": false,
250481
+ "special": true
250482
+ },
250483
+ "50023": {
250484
+ "content": "[STRAINED_BREATH]",
250485
+ "single_word": false,
250486
+ "lstrip": false,
250487
+ "rstrip": false,
250488
+ "normalized": false,
250489
+ "special": true
250490
+ },
250491
+ "50024": {
250492
+ "content": "[QUICK_BREATH]",
250493
+ "single_word": false,
250494
+ "lstrip": false,
250495
+ "rstrip": false,
250496
+ "normalized": false,
250497
+ "special": true
250498
+ },
250499
+ "50025": {
250500
+ "content": "[PANTING]",
250501
+ "single_word": false,
250502
+ "lstrip": false,
250503
+ "rstrip": false,
250504
+ "normalized": false,
250505
+ "special": true
250506
+ },
250507
+ "50026": {
250508
+ "content": "[BREATHING]",
250509
+ "single_word": false,
250510
+ "lstrip": false,
250511
+ "rstrip": false,
250512
+ "normalized": false,
250513
+ "special": true
250514
+ },
250515
+ "50027": {
250516
+ "content": "[HEAVY_BREATHING]",
250517
+ "single_word": false,
250518
+ "lstrip": false,
250519
+ "rstrip": false,
250520
+ "normalized": false,
250521
+ "special": true
250522
+ },
250523
+ "50028": {
250524
+ "content": "[NERVOUS_EXHALE]",
250525
+ "single_word": false,
250526
+ "lstrip": false,
250527
+ "rstrip": false,
250528
+ "normalized": false,
250529
+ "special": true
250530
+ },
250531
+ "50029": {
250532
+ "content": "[PAINED_BREATH]",
250533
+ "single_word": false,
250534
+ "lstrip": false,
250535
+ "rstrip": false,
250536
+ "normalized": false,
250537
+ "special": true
250538
+ },
250539
+ "50030": {
250540
+ "content": "[GIGGLES]",
250541
+ "single_word": false,
250542
+ "lstrip": false,
250543
+ "rstrip": false,
250544
+ "normalized": false,
250545
+ "special": true
250546
+ },
250547
+ "50031": {
250548
+ "content": "[CHUCKLES]",
250549
+ "single_word": false,
250550
+ "lstrip": false,
250551
+ "rstrip": false,
250552
+ "normalized": false,
250553
+ "special": true
250554
+ },
250555
+ "50032": {
250556
+ "content": "[SOFT_CHUCKLE]",
250557
+ "single_word": false,
250558
+ "lstrip": false,
250559
+ "rstrip": false,
250560
+ "normalized": false,
250561
+ "special": true
250562
+ },
250563
+ "50033": {
250564
+ "content": "[SOFT_LAUGH]",
250565
+ "single_word": false,
250566
+ "lstrip": false,
250567
+ "rstrip": false,
250568
+ "normalized": false,
250569
+ "special": true
250570
+ },
250571
+ "50034": {
250572
+ "content": "[NERVOUS_LAUGH]",
250573
+ "single_word": false,
250574
+ "lstrip": false,
250575
+ "rstrip": false,
250576
+ "normalized": false,
250577
+ "special": true
250578
+ },
250579
+ "50035": {
250580
+ "content": "[SOBS]",
250581
+ "single_word": false,
250582
+ "lstrip": false,
250583
+ "rstrip": false,
250584
+ "normalized": false,
250585
+ "special": true
250586
+ },
250587
+ "50036": {
250588
+ "content": "[SOBBING]",
250589
+ "single_word": false,
250590
+ "lstrip": false,
250591
+ "rstrip": false,
250592
+ "normalized": false,
250593
+ "special": true
250594
+ },
250595
+ "50037": {
250596
+ "content": "[CHOKED_SOB]",
250597
+ "single_word": false,
250598
+ "lstrip": false,
250599
+ "rstrip": false,
250600
+ "normalized": false,
250601
+ "special": true
250602
+ },
250603
+ "50038": {
250604
+ "content": "[CRYING]",
250605
+ "single_word": false,
250606
+ "lstrip": false,
250607
+ "rstrip": false,
250608
+ "normalized": false,
250609
+ "special": true
250610
+ },
250611
+ "50039": {
250612
+ "content": "[WHIMPERS]",
250613
+ "single_word": false,
250614
+ "lstrip": false,
250615
+ "rstrip": false,
250616
+ "normalized": false,
250617
+ "special": true
250618
+ },
250619
+ "50040": {
250620
+ "content": "[SOFT_WHIMPER]",
250621
+ "single_word": false,
250622
+ "lstrip": false,
250623
+ "rstrip": false,
250624
+ "normalized": false,
250625
+ "special": true
250626
+ },
250627
+ "50041": {
250628
+ "content": "[PAINED_CRY]",
250629
+ "single_word": false,
250630
+ "lstrip": false,
250631
+ "rstrip": false,
250632
+ "normalized": false,
250633
+ "special": true
250634
+ },
250635
+ "50042": {
250636
+ "content": "[PAINED_GRUNT]",
250637
+ "single_word": false,
250638
+ "lstrip": false,
250639
+ "rstrip": false,
250640
+ "normalized": false,
250641
+ "special": true
250642
+ },
250643
+ "50043": {
250644
+ "content": "[PAINED_SOUND]",
250645
+ "single_word": false,
250646
+ "lstrip": false,
250647
+ "rstrip": false,
250648
+ "normalized": false,
250649
+ "special": true
250650
+ },
250651
+ "50044": {
250652
+ "content": "[CHOKES]",
250653
+ "single_word": false,
250654
+ "lstrip": false,
250655
+ "rstrip": false,
250656
+ "normalized": false,
250657
+ "special": true
250658
+ },
250659
+ "50045": {
250660
+ "content": "[HUMS]",
250661
+ "single_word": false,
250662
+ "lstrip": false,
250663
+ "rstrip": false,
250664
+ "normalized": false,
250665
+ "special": true
250666
+ },
250667
+ "50046": {
250668
+ "content": "[HUMMING]",
250669
+ "single_word": false,
250670
+ "lstrip": false,
250671
+ "rstrip": false,
250672
+ "normalized": false,
250673
+ "special": true
250674
+ },
250675
+ "50047": {
250676
+ "content": "[SOFT_HUM]",
250677
+ "single_word": false,
250678
+ "lstrip": false,
250679
+ "rstrip": false,
250680
+ "normalized": false,
250681
+ "special": true
250682
+ },
250683
+ "50048": {
250684
+ "content": "[PLEASED_HUM]",
250685
+ "single_word": false,
250686
+ "lstrip": false,
250687
+ "rstrip": false,
250688
+ "normalized": false,
250689
+ "special": true
250690
+ },
250691
+ "50049": {
250692
+ "content": "[CONTENTED_HUM]",
250693
+ "single_word": false,
250694
+ "lstrip": false,
250695
+ "rstrip": false,
250696
+ "normalized": false,
250697
+ "special": true
250698
+ },
250699
+ "50050": {
250700
+ "content": "[MURMURS]",
250701
+ "single_word": false,
250702
+ "lstrip": false,
250703
+ "rstrip": false,
250704
+ "normalized": false,
250705
+ "special": true
250706
+ },
250707
+ "50051": {
250708
+ "content": "Mmm",
250709
+ "single_word": false,
250710
+ "lstrip": false,
250711
+ "rstrip": false,
250712
+ "normalized": false,
250713
+ "special": true
250714
+ },
250715
+ "50052": {
250716
+ "content": "Hmm",
250717
+ "single_word": false,
250718
+ "lstrip": false,
250719
+ "rstrip": false,
250720
+ "normalized": false,
250721
+ "special": true
250722
+ },
250723
+ "50053": {
250724
+ "content": "[HMM]",
250725
+ "single_word": false,
250726
+ "lstrip": false,
250727
+ "rstrip": false,
250728
+ "normalized": false,
250729
+ "special": true
250730
+ },
250731
+ "50054": {
250732
+ "content": "[AH]",
250733
+ "single_word": false,
250734
+ "lstrip": false,
250735
+ "rstrip": false,
250736
+ "normalized": false,
250737
+ "special": true
250738
+ },
250739
+ "50055": {
250740
+ "content": "[MOUTH_SOUND]",
250741
+ "single_word": false,
250742
+ "lstrip": false,
250743
+ "rstrip": false,
250744
+ "normalized": false,
250745
+ "special": true
250746
+ },
250747
+ "50056": {
250748
+ "content": "[MOUTH_CLICK]",
250749
+ "single_word": false,
250750
+ "lstrip": false,
250751
+ "rstrip": false,
250752
+ "normalized": false,
250753
+ "special": true
250754
+ },
250755
+ "50057": {
250756
+ "content": "[TONGUE_CLICK]",
250757
+ "single_word": false,
250758
+ "lstrip": false,
250759
+ "rstrip": false,
250760
+ "normalized": false,
250761
+ "special": true
250762
+ },
250763
+ "50058": {
250764
+ "content": "[LIP_SMACK]",
250765
+ "single_word": false,
250766
+ "lstrip": false,
250767
+ "rstrip": false,
250768
+ "normalized": false,
250769
+ "special": true
250770
+ },
250771
+ "50059": {
250772
+ "content": "[SMACKS]",
250773
+ "single_word": false,
250774
+ "lstrip": false,
250775
+ "rstrip": false,
250776
+ "normalized": false,
250777
+ "special": true
250778
+ },
250779
+ "50060": {
250780
+ "content": "[TSK]",
250781
+ "single_word": false,
250782
+ "lstrip": false,
250783
+ "rstrip": false,
250784
+ "normalized": false,
250785
+ "special": true
250786
+ },
250787
+ "50061": {
250788
+ "content": "[CLICKS]",
250789
+ "single_word": false,
250790
+ "lstrip": false,
250791
+ "rstrip": false,
250792
+ "normalized": false,
250793
+ "special": true
250794
+ },
250795
+ "50062": {
250796
+ "content": "[SLURPS]",
250797
+ "single_word": false,
250798
+ "lstrip": false,
250799
+ "rstrip": false,
250800
+ "normalized": false,
250801
+ "special": true
250802
+ },
250803
+ "50063": {
250804
+ "content": "[GULPS]",
250805
+ "single_word": false,
250806
+ "lstrip": false,
250807
+ "rstrip": false,
250808
+ "normalized": false,
250809
+ "special": true
250810
+ },
250811
+ "50064": {
250812
+ "content": "[SWALLOWS]",
250813
+ "single_word": false,
250814
+ "lstrip": false,
250815
+ "rstrip": false,
250816
+ "normalized": false,
250817
+ "special": true
250818
+ },
250819
+ "50065": {
250820
+ "content": "[YAWNS]",
250821
+ "single_word": false,
250822
+ "lstrip": false,
250823
+ "rstrip": false,
250824
+ "normalized": false,
250825
+ "special": true
250826
+ },
250827
+ "50066": {
250828
+ "content": "[MOANS]",
250829
+ "single_word": false,
250830
+ "lstrip": false,
250831
+ "rstrip": false,
250832
+ "normalized": false,
250833
+ "special": true
250834
+ },
250835
+ "50067": {
250836
+ "content": "[GRUNTS]",
250837
+ "single_word": false,
250838
+ "lstrip": false,
250839
+ "rstrip": false,
250840
+ "normalized": false,
250841
+ "special": true
250842
+ },
250843
+ "50068": {
250844
+ "content": "[GROWLS]",
250845
+ "single_word": false,
250846
+ "lstrip": false,
250847
+ "rstrip": false,
250848
+ "normalized": false,
250849
+ "special": true
250850
+ },
250851
+ "50069": {
250852
+ "content": "[HISSES]",
250853
+ "single_word": false,
250854
+ "lstrip": false,
250855
+ "rstrip": false,
250856
+ "normalized": false,
250857
+ "special": true
250858
+ },
250859
+ "50070": {
250860
+ "content": "[SCOFFS]",
250861
+ "single_word": false,
250862
+ "lstrip": false,
250863
+ "rstrip": false,
250864
+ "normalized": false,
250865
+ "special": true
250866
+ },
250867
+ "50071": {
250868
+ "content": "[HMPH]",
250869
+ "single_word": false,
250870
+ "lstrip": false,
250871
+ "rstrip": false,
250872
+ "normalized": false,
250873
+ "special": true
250874
+ },
250875
+ "50072": {
250876
+ "content": "[HUFFS]",
250877
+ "single_word": false,
250878
+ "lstrip": false,
250879
+ "rstrip": false,
250880
+ "normalized": false,
250881
+ "special": true
250882
+ },
250883
+ "50073": {
250884
+ "content": "[WHISPERS]",
250885
+ "single_word": false,
250886
+ "lstrip": false,
250887
+ "rstrip": false,
250888
+ "normalized": false,
250889
+ "special": true
250890
+ },
250891
+ "50074": {
250892
+ "content": "[PURRS]",
250893
+ "single_word": false,
250894
+ "lstrip": false,
250895
+ "rstrip": false,
250896
+ "normalized": false,
250897
+ "special": true
250898
+ },
250899
+ "50075": {
250900
+ "content": "[STUTTERS]",
250901
+ "single_word": false,
250902
+ "lstrip": false,
250903
+ "rstrip": false,
250904
+ "normalized": false,
250905
+ "special": true
250906
+ },
250907
+ "50076": {
250908
+ "content": "[HICCUPS]",
250909
+ "single_word": false,
250910
+ "lstrip": false,
250911
+ "rstrip": false,
250912
+ "normalized": false,
250913
+ "special": true
250914
+ },
250915
+ "50077": {
250916
+ "content": "[SHIVERS]",
250917
+ "single_word": false,
250918
+ "lstrip": false,
250919
+ "rstrip": false,
250920
+ "normalized": false,
250921
+ "special": true
250922
+ },
250923
+ "50078": {
250924
+ "content": "[YELPS]",
250925
+ "single_word": false,
250926
+ "lstrip": false,
250927
+ "rstrip": false,
250928
+ "normalized": false,
250929
+ "special": true
250930
+ },
250931
+ "50079": {
250932
+ "content": "[ROARS]",
250933
+ "single_word": false,
250934
+ "lstrip": false,
250935
+ "rstrip": false,
250936
+ "normalized": false,
250937
+ "special": true
250938
+ }
250939
  }
250940
  }
tokenizer_config.json CHANGED
@@ -90,10 +90,90 @@
90
  "<|reserved_77|>",
91
  "<|reserved_78|>",
92
  "<|reserved_79|>",
93
- "<|reserved_80|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
  ],
95
  "model_max_length": 1000000000000000019884624838656,
96
  "pad_token": "<|pad|>",
97
  "tokenizer_class": "TokenizersBackend",
98
  "unk_token": "<|unk|>"
99
- }
 
90
  "<|reserved_77|>",
91
  "<|reserved_78|>",
92
  "<|reserved_79|>",
93
+ "<|reserved_80|>",
94
+ "[LAUGHS]",
95
+ "[SIGHS]",
96
+ "[GASPS]",
97
+ "[CLEARS_THROAT]",
98
+ "[MUSIC]",
99
+ "[BEEP]",
100
+ "[APPLAUSE]",
101
+ "[COUGHS]",
102
+ "[MUMBLES]",
103
+ "[WHISTLES]",
104
+ "[SNIFFS]",
105
+ "[SINGS]",
106
+ "[GROANS]",
107
+ "[SCREAMS]",
108
+ "[SNEEZES]",
109
+ "[EXHALES]",
110
+ "[INHALES]",
111
+ "[BREATH]",
112
+ "[DEEP_BREATH]",
113
+ "[SHARP_INHALE]",
114
+ "[SHARP_BREATH]",
115
+ "[SHAKY_BREATH]",
116
+ "[BREATHY_EXHALE]",
117
+ "[STRAINED_BREATH]",
118
+ "[QUICK_BREATH]",
119
+ "[PANTING]",
120
+ "[BREATHING]",
121
+ "[HEAVY_BREATHING]",
122
+ "[NERVOUS_EXHALE]",
123
+ "[PAINED_BREATH]",
124
+ "[GIGGLES]",
125
+ "[CHUCKLES]",
126
+ "[SOFT_CHUCKLE]",
127
+ "[SOFT_LAUGH]",
128
+ "[NERVOUS_LAUGH]",
129
+ "[SOBS]",
130
+ "[SOBBING]",
131
+ "[CHOKED_SOB]",
132
+ "[CRYING]",
133
+ "[WHIMPERS]",
134
+ "[SOFT_WHIMPER]",
135
+ "[PAINED_CRY]",
136
+ "[PAINED_GRUNT]",
137
+ "[PAINED_SOUND]",
138
+ "[CHOKES]",
139
+ "[HUMS]",
140
+ "[HUMMING]",
141
+ "[SOFT_HUM]",
142
+ "[PLEASED_HUM]",
143
+ "[CONTENTED_HUM]",
144
+ "[MURMURS]",
145
+ "Mmm",
146
+ "Hmm",
147
+ "[HMM]",
148
+ "[AH]",
149
+ "[MOUTH_SOUND]",
150
+ "[MOUTH_CLICK]",
151
+ "[TONGUE_CLICK]",
152
+ "[LIP_SMACK]",
153
+ "[SMACKS]",
154
+ "[TSK]",
155
+ "[CLICKS]",
156
+ "[SLURPS]",
157
+ "[GULPS]",
158
+ "[SWALLOWS]",
159
+ "[YAWNS]",
160
+ "[MOANS]",
161
+ "[GRUNTS]",
162
+ "[GROWLS]",
163
+ "[HISSES]",
164
+ "[SCOFFS]",
165
+ "[HMPH]",
166
+ "[HUFFS]",
167
+ "[WHISPERS]",
168
+ "[PURRS]",
169
+ "[STUTTERS]",
170
+ "[HICCUPS]",
171
+ "[SHIVERS]",
172
+ "[YELPS]",
173
+ "[ROARS]"
174
  ],
175
  "model_max_length": 1000000000000000019884624838656,
176
  "pad_token": "<|pad|>",
177
  "tokenizer_class": "TokenizersBackend",
178
  "unk_token": "<|unk|>"
179
+ }