0x8badbeef commited on 8 days ago

Commit

d0002ff

verified ·

1 Parent(s): 4c54f5f

Upload folder using huggingface_hub

Browse files

Files changed (24) hide show

.gitattributes +1 -0
added_tokens.json +2483 -0
audio_modules.pt +3 -0
chat_template.jinja +13 -0
config.json +44 -0
config_molmo.py +60 -0
generation_config.json +4 -0
image_preprocessing_molmo.py +547 -0
merges.txt +0 -0
model-00001-of-00005.safetensors +3 -0
model-00002-of-00005.safetensors +3 -0
model-00003-of-00005.safetensors +3 -0
model-00004-of-00005.safetensors +3 -0
model-00005-of-00005.safetensors +3 -0
model.safetensors.index.json +592 -0
modeling_molmo.py +2371 -0
preprocessing_molmo.py +192 -0
preprocessor_config.json +32 -0
processor_config.json +6 -0
special_tokens_map.json +2073 -0
token_manifest.json +14 -0
tokenizer.json +3 -0
tokenizer_config.json +0 -0
vocab.json +0 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json ADDED Viewed

	@@ -0,0 +1,2483 @@

+{
+  "<audio_0>": 152072,
+  "<audio_1000>": 153072,
+  "<audio_1001>": 153073,
+  "<audio_1002>": 153074,
+  "<audio_1003>": 153075,
+  "<audio_1004>": 153076,
+  "<audio_1005>": 153077,
+  "<audio_1006>": 153078,
+  "<audio_1007>": 153079,
+  "<audio_1008>": 153080,
+  "<audio_1009>": 153081,
+  "<audio_100>": 152172,
+  "<audio_1010>": 153082,
+  "<audio_1011>": 153083,
+  "<audio_1012>": 153084,
+  "<audio_1013>": 153085,
+  "<audio_1014>": 153086,
+  "<audio_1015>": 153087,
+  "<audio_1016>": 153088,
+  "<audio_1017>": 153089,
+  "<audio_1018>": 153090,
+  "<audio_1019>": 153091,
+  "<audio_101>": 152173,
+  "<audio_1020>": 153092,
+  "<audio_1021>": 153093,
+  "<audio_1022>": 153094,
+  "<audio_1023>": 153095,
+  "<audio_1024>": 153096,
+  "<audio_1025>": 153097,
+  "<audio_1026>": 153098,
+  "<audio_1027>": 153099,
+  "<audio_1028>": 153100,
+  "<audio_1029>": 153101,
+  "<audio_102>": 152174,
+  "<audio_1030>": 153102,
+  "<audio_1031>": 153103,
+  "<audio_1032>": 153104,
+  "<audio_1033>": 153105,
+  "<audio_1034>": 153106,
+  "<audio_1035>": 153107,
+  "<audio_1036>": 153108,
+  "<audio_1037>": 153109,
+  "<audio_1038>": 153110,
+  "<audio_1039>": 153111,
+  "<audio_103>": 152175,
+  "<audio_1040>": 153112,
+  "<audio_1041>": 153113,
+  "<audio_1042>": 153114,
+  "<audio_1043>": 153115,
+  "<audio_1044>": 153116,
+  "<audio_1045>": 153117,
+  "<audio_1046>": 153118,
+  "<audio_1047>": 153119,
+  "<audio_1048>": 153120,
+  "<audio_1049>": 153121,
+  "<audio_104>": 152176,
+  "<audio_1050>": 153122,
+  "<audio_1051>": 153123,
+  "<audio_1052>": 153124,
+  "<audio_1053>": 153125,
+  "<audio_1054>": 153126,
+  "<audio_1055>": 153127,
+  "<audio_1056>": 153128,
+  "<audio_1057>": 153129,
+  "<audio_1058>": 153130,
+  "<audio_1059>": 153131,
+  "<audio_105>": 152177,
+  "<audio_1060>": 153132,
+  "<audio_1061>": 153133,
+  "<audio_1062>": 153134,
+  "<audio_1063>": 153135,
+  "<audio_1064>": 153136,
+  "<audio_1065>": 153137,
+  "<audio_1066>": 153138,
+  "<audio_1067>": 153139,
+  "<audio_1068>": 153140,
+  "<audio_1069>": 153141,
+  "<audio_106>": 152178,
+  "<audio_1070>": 153142,
+  "<audio_1071>": 153143,
+  "<audio_1072>": 153144,
+  "<audio_1073>": 153145,
+  "<audio_1074>": 153146,
+  "<audio_1075>": 153147,
+  "<audio_1076>": 153148,
+  "<audio_1077>": 153149,
+  "<audio_1078>": 153150,
+  "<audio_1079>": 153151,
+  "<audio_107>": 152179,
+  "<audio_1080>": 153152,
+  "<audio_1081>": 153153,
+  "<audio_1082>": 153154,
+  "<audio_1083>": 153155,
+  "<audio_1084>": 153156,
+  "<audio_1085>": 153157,
+  "<audio_1086>": 153158,
+  "<audio_1087>": 153159,
+  "<audio_1088>": 153160,
+  "<audio_1089>": 153161,
+  "<audio_108>": 152180,
+  "<audio_1090>": 153162,
+  "<audio_1091>": 153163,
+  "<audio_1092>": 153164,
+  "<audio_1093>": 153165,
+  "<audio_1094>": 153166,
+  "<audio_1095>": 153167,
+  "<audio_1096>": 153168,
+  "<audio_1097>": 153169,
+  "<audio_1098>": 153170,
+  "<audio_1099>": 153171,
+  "<audio_109>": 152181,
+  "<audio_10>": 152082,
+  "<audio_1100>": 153172,
+  "<audio_1101>": 153173,
+  "<audio_1102>": 153174,
+  "<audio_1103>": 153175,
+  "<audio_1104>": 153176,
+  "<audio_1105>": 153177,
+  "<audio_1106>": 153178,
+  "<audio_1107>": 153179,
+  "<audio_1108>": 153180,
+  "<audio_1109>": 153181,
+  "<audio_110>": 152182,
+  "<audio_1110>": 153182,
+  "<audio_1111>": 153183,
+  "<audio_1112>": 153184,
+  "<audio_1113>": 153185,
+  "<audio_1114>": 153186,
+  "<audio_1115>": 153187,
+  "<audio_1116>": 153188,
+  "<audio_1117>": 153189,
+  "<audio_1118>": 153190,
+  "<audio_1119>": 153191,
+  "<audio_111>": 152183,
+  "<audio_1120>": 153192,
+  "<audio_1121>": 153193,
+  "<audio_1122>": 153194,
+  "<audio_1123>": 153195,
+  "<audio_1124>": 153196,
+  "<audio_1125>": 153197,
+  "<audio_1126>": 153198,
+  "<audio_1127>": 153199,
+  "<audio_1128>": 153200,
+  "<audio_1129>": 153201,
+  "<audio_112>": 152184,
+  "<audio_1130>": 153202,
+  "<audio_1131>": 153203,
+  "<audio_1132>": 153204,
+  "<audio_1133>": 153205,
+  "<audio_1134>": 153206,
+  "<audio_1135>": 153207,
+  "<audio_1136>": 153208,
+  "<audio_1137>": 153209,
+  "<audio_1138>": 153210,
+  "<audio_1139>": 153211,
+  "<audio_113>": 152185,
+  "<audio_1140>": 153212,
+  "<audio_1141>": 153213,
+  "<audio_1142>": 153214,
+  "<audio_1143>": 153215,
+  "<audio_1144>": 153216,
+  "<audio_1145>": 153217,
+  "<audio_1146>": 153218,
+  "<audio_1147>": 153219,
+  "<audio_1148>": 153220,
+  "<audio_1149>": 153221,
+  "<audio_114>": 152186,
+  "<audio_1150>": 153222,
+  "<audio_1151>": 153223,
+  "<audio_1152>": 153224,
+  "<audio_1153>": 153225,
+  "<audio_1154>": 153226,
+  "<audio_1155>": 153227,
+  "<audio_1156>": 153228,
+  "<audio_1157>": 153229,
+  "<audio_1158>": 153230,
+  "<audio_1159>": 153231,
+  "<audio_115>": 152187,
+  "<audio_1160>": 153232,
+  "<audio_1161>": 153233,
+  "<audio_1162>": 153234,
+  "<audio_1163>": 153235,
+  "<audio_1164>": 153236,
+  "<audio_1165>": 153237,
+  "<audio_1166>": 153238,
+  "<audio_1167>": 153239,
+  "<audio_1168>": 153240,
+  "<audio_1169>": 153241,
+  "<audio_116>": 152188,
+  "<audio_1170>": 153242,
+  "<audio_1171>": 153243,
+  "<audio_1172>": 153244,
+  "<audio_1173>": 153245,
+  "<audio_1174>": 153246,
+  "<audio_1175>": 153247,
+  "<audio_1176>": 153248,
+  "<audio_1177>": 153249,
+  "<audio_1178>": 153250,
+  "<audio_1179>": 153251,
+  "<audio_117>": 152189,
+  "<audio_1180>": 153252,
+  "<audio_1181>": 153253,
+  "<audio_1182>": 153254,
+  "<audio_1183>": 153255,
+  "<audio_1184>": 153256,
+  "<audio_1185>": 153257,
+  "<audio_1186>": 153258,
+  "<audio_1187>": 153259,
+  "<audio_1188>": 153260,
+  "<audio_1189>": 153261,
+  "<audio_118>": 152190,
+  "<audio_1190>": 153262,
+  "<audio_1191>": 153263,
+  "<audio_1192>": 153264,
+  "<audio_1193>": 153265,
+  "<audio_1194>": 153266,
+  "<audio_1195>": 153267,
+  "<audio_1196>": 153268,
+  "<audio_1197>": 153269,
+  "<audio_1198>": 153270,
+  "<audio_1199>": 153271,
+  "<audio_119>": 152191,
+  "<audio_11>": 152083,
+  "<audio_1200>": 153272,
+  "<audio_1201>": 153273,
+  "<audio_1202>": 153274,
+  "<audio_1203>": 153275,
+  "<audio_1204>": 153276,
+  "<audio_1205>": 153277,
+  "<audio_1206>": 153278,
+  "<audio_1207>": 153279,
+  "<audio_1208>": 153280,
+  "<audio_1209>": 153281,
+  "<audio_120>": 152192,
+  "<audio_1210>": 153282,
+  "<audio_1211>": 153283,
+  "<audio_1212>": 153284,
+  "<audio_1213>": 153285,
+  "<audio_1214>": 153286,
+  "<audio_1215>": 153287,
+  "<audio_1216>": 153288,
+  "<audio_1217>": 153289,
+  "<audio_1218>": 153290,
+  "<audio_1219>": 153291,
+  "<audio_121>": 152193,
+  "<audio_1220>": 153292,
+  "<audio_1221>": 153293,
+  "<audio_1222>": 153294,
+  "<audio_1223>": 153295,
+  "<audio_1224>": 153296,
+  "<audio_1225>": 153297,
+  "<audio_1226>": 153298,
+  "<audio_1227>": 153299,
+  "<audio_1228>": 153300,
+  "<audio_1229>": 153301,
+  "<audio_122>": 152194,
+  "<audio_1230>": 153302,
+  "<audio_1231>": 153303,
+  "<audio_1232>": 153304,
+  "<audio_1233>": 153305,
+  "<audio_1234>": 153306,
+  "<audio_1235>": 153307,
+  "<audio_1236>": 153308,
+  "<audio_1237>": 153309,
+  "<audio_1238>": 153310,
+  "<audio_1239>": 153311,
+  "<audio_123>": 152195,
+  "<audio_1240>": 153312,
+  "<audio_1241>": 153313,
+  "<audio_1242>": 153314,
+  "<audio_1243>": 153315,
+  "<audio_1244>": 153316,
+  "<audio_1245>": 153317,
+  "<audio_1246>": 153318,
+  "<audio_1247>": 153319,
+  "<audio_1248>": 153320,
+  "<audio_1249>": 153321,
+  "<audio_124>": 152196,
+  "<audio_1250>": 153322,
+  "<audio_1251>": 153323,
+  "<audio_1252>": 153324,
+  "<audio_1253>": 153325,
+  "<audio_1254>": 153326,
+  "<audio_1255>": 153327,
+  "<audio_1256>": 153328,
+  "<audio_1257>": 153329,
+  "<audio_1258>": 153330,
+  "<audio_1259>": 153331,
+  "<audio_125>": 152197,
+  "<audio_1260>": 153332,
+  "<audio_1261>": 153333,
+  "<audio_1262>": 153334,
+  "<audio_1263>": 153335,
+  "<audio_1264>": 153336,
+  "<audio_1265>": 153337,
+  "<audio_1266>": 153338,
+  "<audio_1267>": 153339,
+  "<audio_1268>": 153340,
+  "<audio_1269>": 153341,
+  "<audio_126>": 152198,
+  "<audio_1270>": 153342,
+  "<audio_1271>": 153343,
+  "<audio_1272>": 153344,
+  "<audio_1273>": 153345,
+  "<audio_1274>": 153346,
+  "<audio_1275>": 153347,
+  "<audio_1276>": 153348,
+  "<audio_1277>": 153349,
+  "<audio_1278>": 153350,
+  "<audio_1279>": 153351,
+  "<audio_127>": 152199,
+  "<audio_1280>": 153352,
+  "<audio_1281>": 153353,
+  "<audio_1282>": 153354,
+  "<audio_1283>": 153355,
+  "<audio_1284>": 153356,
+  "<audio_1285>": 153357,
+  "<audio_1286>": 153358,
+  "<audio_1287>": 153359,
+  "<audio_1288>": 153360,
+  "<audio_1289>": 153361,
+  "<audio_128>": 152200,
+  "<audio_1290>": 153362,
+  "<audio_1291>": 153363,
+  "<audio_1292>": 153364,
+  "<audio_1293>": 153365,
+  "<audio_1294>": 153366,
+  "<audio_1295>": 153367,
+  "<audio_1296>": 153368,
+  "<audio_1297>": 153369,
+  "<audio_1298>": 153370,
+  "<audio_1299>": 153371,
+  "<audio_129>": 152201,
+  "<audio_12>": 152084,
+  "<audio_1300>": 153372,
+  "<audio_1301>": 153373,
+  "<audio_1302>": 153374,
+  "<audio_1303>": 153375,
+  "<audio_1304>": 153376,
+  "<audio_1305>": 153377,
+  "<audio_1306>": 153378,
+  "<audio_1307>": 153379,
+  "<audio_1308>": 153380,
+  "<audio_1309>": 153381,
+  "<audio_130>": 152202,
+  "<audio_1310>": 153382,
+  "<audio_1311>": 153383,
+  "<audio_1312>": 153384,
+  "<audio_1313>": 153385,
+  "<audio_1314>": 153386,
+  "<audio_1315>": 153387,
+  "<audio_1316>": 153388,
+  "<audio_1317>": 153389,
+  "<audio_1318>": 153390,
+  "<audio_1319>": 153391,
+  "<audio_131>": 152203,
+  "<audio_1320>": 153392,
+  "<audio_1321>": 153393,
+  "<audio_1322>": 153394,
+  "<audio_1323>": 153395,
+  "<audio_1324>": 153396,
+  "<audio_1325>": 153397,
+  "<audio_1326>": 153398,
+  "<audio_1327>": 153399,
+  "<audio_1328>": 153400,
+  "<audio_1329>": 153401,
+  "<audio_132>": 152204,
+  "<audio_1330>": 153402,
+  "<audio_1331>": 153403,
+  "<audio_1332>": 153404,
+  "<audio_1333>": 153405,
+  "<audio_1334>": 153406,
+  "<audio_1335>": 153407,
+  "<audio_1336>": 153408,
+  "<audio_1337>": 153409,
+  "<audio_1338>": 153410,
+  "<audio_1339>": 153411,
+  "<audio_133>": 152205,
+  "<audio_1340>": 153412,
+  "<audio_1341>": 153413,
+  "<audio_1342>": 153414,
+  "<audio_1343>": 153415,
+  "<audio_1344>": 153416,
+  "<audio_1345>": 153417,
+  "<audio_1346>": 153418,
+  "<audio_1347>": 153419,
+  "<audio_1348>": 153420,
+  "<audio_1349>": 153421,
+  "<audio_134>": 152206,
+  "<audio_1350>": 153422,
+  "<audio_1351>": 153423,
+  "<audio_1352>": 153424,
+  "<audio_1353>": 153425,
+  "<audio_1354>": 153426,
+  "<audio_1355>": 153427,
+  "<audio_1356>": 153428,
+  "<audio_1357>": 153429,
+  "<audio_1358>": 153430,
+  "<audio_1359>": 153431,
+  "<audio_135>": 152207,
+  "<audio_1360>": 153432,
+  "<audio_1361>": 153433,
+  "<audio_1362>": 153434,
+  "<audio_1363>": 153435,
+  "<audio_1364>": 153436,
+  "<audio_1365>": 153437,
+  "<audio_1366>": 153438,
+  "<audio_1367>": 153439,
+  "<audio_1368>": 153440,
+  "<audio_1369>": 153441,
+  "<audio_136>": 152208,
+  "<audio_1370>": 153442,
+  "<audio_1371>": 153443,
+  "<audio_1372>": 153444,
+  "<audio_1373>": 153445,
+  "<audio_1374>": 153446,
+  "<audio_1375>": 153447,
+  "<audio_1376>": 153448,
+  "<audio_1377>": 153449,
+  "<audio_1378>": 153450,
+  "<audio_1379>": 153451,
+  "<audio_137>": 152209,
+  "<audio_1380>": 153452,
+  "<audio_1381>": 153453,
+  "<audio_1382>": 153454,
+  "<audio_1383>": 153455,
+  "<audio_1384>": 153456,
+  "<audio_1385>": 153457,
+  "<audio_1386>": 153458,
+  "<audio_1387>": 153459,
+  "<audio_1388>": 153460,
+  "<audio_1389>": 153461,
+  "<audio_138>": 152210,
+  "<audio_1390>": 153462,
+  "<audio_1391>": 153463,
+  "<audio_1392>": 153464,
+  "<audio_1393>": 153465,
+  "<audio_1394>": 153466,
+  "<audio_1395>": 153467,
+  "<audio_1396>": 153468,
+  "<audio_1397>": 153469,
+  "<audio_1398>": 153470,
+  "<audio_1399>": 153471,
+  "<audio_139>": 152211,
+  "<audio_13>": 152085,
+  "<audio_1400>": 153472,
+  "<audio_1401>": 153473,
+  "<audio_1402>": 153474,
+  "<audio_1403>": 153475,
+  "<audio_1404>": 153476,
+  "<audio_1405>": 153477,
+  "<audio_1406>": 153478,
+  "<audio_1407>": 153479,
+  "<audio_1408>": 153480,
+  "<audio_1409>": 153481,
+  "<audio_140>": 152212,
+  "<audio_1410>": 153482,
+  "<audio_1411>": 153483,
+  "<audio_1412>": 153484,
+  "<audio_1413>": 153485,
+  "<audio_1414>": 153486,
+  "<audio_1415>": 153487,
+  "<audio_1416>": 153488,
+  "<audio_1417>": 153489,
+  "<audio_1418>": 153490,
+  "<audio_1419>": 153491,
+  "<audio_141>": 152213,
+  "<audio_1420>": 153492,
+  "<audio_1421>": 153493,
+  "<audio_1422>": 153494,
+  "<audio_1423>": 153495,
+  "<audio_1424>": 153496,
+  "<audio_1425>": 153497,
+  "<audio_1426>": 153498,
+  "<audio_1427>": 153499,
+  "<audio_1428>": 153500,
+  "<audio_1429>": 153501,
+  "<audio_142>": 152214,
+  "<audio_1430>": 153502,
+  "<audio_1431>": 153503,
+  "<audio_1432>": 153504,
+  "<audio_1433>": 153505,
+  "<audio_1434>": 153506,
+  "<audio_1435>": 153507,
+  "<audio_1436>": 153508,
+  "<audio_1437>": 153509,
+  "<audio_1438>": 153510,
+  "<audio_1439>": 153511,
+  "<audio_143>": 152215,
+  "<audio_1440>": 153512,
+  "<audio_1441>": 153513,
+  "<audio_1442>": 153514,
+  "<audio_1443>": 153515,
+  "<audio_1444>": 153516,
+  "<audio_1445>": 153517,
+  "<audio_1446>": 153518,
+  "<audio_1447>": 153519,
+  "<audio_1448>": 153520,
+  "<audio_1449>": 153521,
+  "<audio_144>": 152216,
+  "<audio_1450>": 153522,
+  "<audio_1451>": 153523,
+  "<audio_1452>": 153524,
+  "<audio_1453>": 153525,
+  "<audio_1454>": 153526,
+  "<audio_1455>": 153527,
+  "<audio_1456>": 153528,
+  "<audio_1457>": 153529,
+  "<audio_1458>": 153530,
+  "<audio_1459>": 153531,
+  "<audio_145>": 152217,
+  "<audio_1460>": 153532,
+  "<audio_1461>": 153533,
+  "<audio_1462>": 153534,
+  "<audio_1463>": 153535,
+  "<audio_1464>": 153536,
+  "<audio_1465>": 153537,
+  "<audio_1466>": 153538,
+  "<audio_1467>": 153539,
+  "<audio_1468>": 153540,
+  "<audio_1469>": 153541,
+  "<audio_146>": 152218,
+  "<audio_1470>": 153542,
+  "<audio_1471>": 153543,
+  "<audio_1472>": 153544,
+  "<audio_1473>": 153545,
+  "<audio_1474>": 153546,
+  "<audio_1475>": 153547,
+  "<audio_1476>": 153548,
+  "<audio_1477>": 153549,
+  "<audio_1478>": 153550,
+  "<audio_1479>": 153551,
+  "<audio_147>": 152219,
+  "<audio_1480>": 153552,
+  "<audio_1481>": 153553,
+  "<audio_1482>": 153554,
+  "<audio_1483>": 153555,
+  "<audio_1484>": 153556,
+  "<audio_1485>": 153557,
+  "<audio_1486>": 153558,
+  "<audio_1487>": 153559,
+  "<audio_1488>": 153560,
+  "<audio_1489>": 153561,
+  "<audio_148>": 152220,
+  "<audio_1490>": 153562,
+  "<audio_1491>": 153563,
+  "<audio_1492>": 153564,
+  "<audio_1493>": 153565,
+  "<audio_1494>": 153566,
+  "<audio_1495>": 153567,
+  "<audio_1496>": 153568,
+  "<audio_1497>": 153569,
+  "<audio_1498>": 153570,
+  "<audio_1499>": 153571,
+  "<audio_149>": 152221,
+  "<audio_14>": 152086,
+  "<audio_1500>": 153572,
+  "<audio_1501>": 153573,
+  "<audio_1502>": 153574,
+  "<audio_1503>": 153575,
+  "<audio_1504>": 153576,
+  "<audio_1505>": 153577,
+  "<audio_1506>": 153578,
+  "<audio_1507>": 153579,
+  "<audio_1508>": 153580,
+  "<audio_1509>": 153581,
+  "<audio_150>": 152222,
+  "<audio_1510>": 153582,
+  "<audio_1511>": 153583,
+  "<audio_1512>": 153584,
+  "<audio_1513>": 153585,
+  "<audio_1514>": 153586,
+  "<audio_1515>": 153587,
+  "<audio_1516>": 153588,
+  "<audio_1517>": 153589,
+  "<audio_1518>": 153590,
+  "<audio_1519>": 153591,
+  "<audio_151>": 152223,
+  "<audio_1520>": 153592,
+  "<audio_1521>": 153593,
+  "<audio_1522>": 153594,
+  "<audio_1523>": 153595,
+  "<audio_1524>": 153596,
+  "<audio_1525>": 153597,
+  "<audio_1526>": 153598,
+  "<audio_1527>": 153599,
+  "<audio_1528>": 153600,
+  "<audio_1529>": 153601,
+  "<audio_152>": 152224,
+  "<audio_1530>": 153602,
+  "<audio_1531>": 153603,
+  "<audio_1532>": 153604,
+  "<audio_1533>": 153605,
+  "<audio_1534>": 153606,
+  "<audio_1535>": 153607,
+  "<audio_1536>": 153608,
+  "<audio_1537>": 153609,
+  "<audio_1538>": 153610,
+  "<audio_1539>": 153611,
+  "<audio_153>": 152225,
+  "<audio_1540>": 153612,
+  "<audio_1541>": 153613,
+  "<audio_1542>": 153614,
+  "<audio_1543>": 153615,
+  "<audio_1544>": 153616,
+  "<audio_1545>": 153617,
+  "<audio_1546>": 153618,
+  "<audio_1547>": 153619,
+  "<audio_1548>": 153620,
+  "<audio_1549>": 153621,
+  "<audio_154>": 152226,
+  "<audio_1550>": 153622,
+  "<audio_1551>": 153623,
+  "<audio_1552>": 153624,
+  "<audio_1553>": 153625,
+  "<audio_1554>": 153626,
+  "<audio_1555>": 153627,
+  "<audio_1556>": 153628,
+  "<audio_1557>": 153629,
+  "<audio_1558>": 153630,
+  "<audio_1559>": 153631,
+  "<audio_155>": 152227,
+  "<audio_1560>": 153632,
+  "<audio_1561>": 153633,
+  "<audio_1562>": 153634,
+  "<audio_1563>": 153635,
+  "<audio_1564>": 153636,
+  "<audio_1565>": 153637,
+  "<audio_1566>": 153638,
+  "<audio_1567>": 153639,
+  "<audio_1568>": 153640,
+  "<audio_1569>": 153641,
+  "<audio_156>": 152228,
+  "<audio_1570>": 153642,
+  "<audio_1571>": 153643,
+  "<audio_1572>": 153644,
+  "<audio_1573>": 153645,
+  "<audio_1574>": 153646,
+  "<audio_1575>": 153647,
+  "<audio_1576>": 153648,
+  "<audio_1577>": 153649,
+  "<audio_1578>": 153650,
+  "<audio_1579>": 153651,
+  "<audio_157>": 152229,
+  "<audio_1580>": 153652,
+  "<audio_1581>": 153653,
+  "<audio_1582>": 153654,
+  "<audio_1583>": 153655,
+  "<audio_1584>": 153656,
+  "<audio_1585>": 153657,
+  "<audio_1586>": 153658,
+  "<audio_1587>": 153659,
+  "<audio_1588>": 153660,
+  "<audio_1589>": 153661,
+  "<audio_158>": 152230,
+  "<audio_1590>": 153662,
+  "<audio_1591>": 153663,
+  "<audio_1592>": 153664,
+  "<audio_1593>": 153665,
+  "<audio_1594>": 153666,
+  "<audio_1595>": 153667,
+  "<audio_1596>": 153668,
+  "<audio_1597>": 153669,
+  "<audio_1598>": 153670,
+  "<audio_1599>": 153671,
+  "<audio_159>": 152231,
+  "<audio_15>": 152087,
+  "<audio_1600>": 153672,
+  "<audio_1601>": 153673,
+  "<audio_1602>": 153674,
+  "<audio_1603>": 153675,
+  "<audio_1604>": 153676,
+  "<audio_1605>": 153677,
+  "<audio_1606>": 153678,
+  "<audio_1607>": 153679,
+  "<audio_1608>": 153680,
+  "<audio_1609>": 153681,
+  "<audio_160>": 152232,
+  "<audio_1610>": 153682,
+  "<audio_1611>": 153683,
+  "<audio_1612>": 153684,
+  "<audio_1613>": 153685,
+  "<audio_1614>": 153686,
+  "<audio_1615>": 153687,
+  "<audio_1616>": 153688,
+  "<audio_1617>": 153689,
+  "<audio_1618>": 153690,
+  "<audio_1619>": 153691,
+  "<audio_161>": 152233,
+  "<audio_1620>": 153692,
+  "<audio_1621>": 153693,
+  "<audio_1622>": 153694,
+  "<audio_1623>": 153695,
+  "<audio_1624>": 153696,
+  "<audio_1625>": 153697,
+  "<audio_1626>": 153698,
+  "<audio_1627>": 153699,
+  "<audio_1628>": 153700,
+  "<audio_1629>": 153701,
+  "<audio_162>": 152234,
+  "<audio_1630>": 153702,
+  "<audio_1631>": 153703,
+  "<audio_1632>": 153704,
+  "<audio_1633>": 153705,
+  "<audio_1634>": 153706,
+  "<audio_1635>": 153707,
+  "<audio_1636>": 153708,
+  "<audio_1637>": 153709,
+  "<audio_1638>": 153710,
+  "<audio_1639>": 153711,
+  "<audio_163>": 152235,
+  "<audio_1640>": 153712,
+  "<audio_1641>": 153713,
+  "<audio_1642>": 153714,
+  "<audio_1643>": 153715,
+  "<audio_1644>": 153716,
+  "<audio_1645>": 153717,
+  "<audio_1646>": 153718,
+  "<audio_1647>": 153719,
+  "<audio_1648>": 153720,
+  "<audio_1649>": 153721,
+  "<audio_164>": 152236,
+  "<audio_1650>": 153722,
+  "<audio_1651>": 153723,
+  "<audio_1652>": 153724,
+  "<audio_1653>": 153725,
+  "<audio_1654>": 153726,
+  "<audio_1655>": 153727,
+  "<audio_1656>": 153728,
+  "<audio_1657>": 153729,
+  "<audio_1658>": 153730,
+  "<audio_1659>": 153731,
+  "<audio_165>": 152237,
+  "<audio_1660>": 153732,
+  "<audio_1661>": 153733,
+  "<audio_1662>": 153734,
+  "<audio_1663>": 153735,
+  "<audio_1664>": 153736,
+  "<audio_1665>": 153737,
+  "<audio_1666>": 153738,
+  "<audio_1667>": 153739,
+  "<audio_1668>": 153740,
+  "<audio_1669>": 153741,
+  "<audio_166>": 152238,
+  "<audio_1670>": 153742,
+  "<audio_1671>": 153743,
+  "<audio_1672>": 153744,
+  "<audio_1673>": 153745,
+  "<audio_1674>": 153746,
+  "<audio_1675>": 153747,
+  "<audio_1676>": 153748,
+  "<audio_1677>": 153749,
+  "<audio_1678>": 153750,
+  "<audio_1679>": 153751,
+  "<audio_167>": 152239,
+  "<audio_1680>": 153752,
+  "<audio_1681>": 153753,
+  "<audio_1682>": 153754,
+  "<audio_1683>": 153755,
+  "<audio_1684>": 153756,
+  "<audio_1685>": 153757,
+  "<audio_1686>": 153758,
+  "<audio_1687>": 153759,
+  "<audio_1688>": 153760,
+  "<audio_1689>": 153761,
+  "<audio_168>": 152240,
+  "<audio_1690>": 153762,
+  "<audio_1691>": 153763,
+  "<audio_1692>": 153764,
+  "<audio_1693>": 153765,
+  "<audio_1694>": 153766,
+  "<audio_1695>": 153767,
+  "<audio_1696>": 153768,
+  "<audio_1697>": 153769,
+  "<audio_1698>": 153770,
+  "<audio_1699>": 153771,
+  "<audio_169>": 152241,
+  "<audio_16>": 152088,
+  "<audio_1700>": 153772,
+  "<audio_1701>": 153773,
+  "<audio_1702>": 153774,
+  "<audio_1703>": 153775,
+  "<audio_1704>": 153776,
+  "<audio_1705>": 153777,
+  "<audio_1706>": 153778,
+  "<audio_1707>": 153779,
+  "<audio_1708>": 153780,
+  "<audio_1709>": 153781,
+  "<audio_170>": 152242,
+  "<audio_1710>": 153782,
+  "<audio_1711>": 153783,
+  "<audio_1712>": 153784,
+  "<audio_1713>": 153785,
+  "<audio_1714>": 153786,
+  "<audio_1715>": 153787,
+  "<audio_1716>": 153788,
+  "<audio_1717>": 153789,
+  "<audio_1718>": 153790,
+  "<audio_1719>": 153791,
+  "<audio_171>": 152243,
+  "<audio_1720>": 153792,
+  "<audio_1721>": 153793,
+  "<audio_1722>": 153794,
+  "<audio_1723>": 153795,
+  "<audio_1724>": 153796,
+  "<audio_1725>": 153797,
+  "<audio_1726>": 153798,
+  "<audio_1727>": 153799,
+  "<audio_1728>": 153800,
+  "<audio_1729>": 153801,
+  "<audio_172>": 152244,
+  "<audio_1730>": 153802,
+  "<audio_1731>": 153803,
+  "<audio_1732>": 153804,
+  "<audio_1733>": 153805,
+  "<audio_1734>": 153806,
+  "<audio_1735>": 153807,
+  "<audio_1736>": 153808,
+  "<audio_1737>": 153809,
+  "<audio_1738>": 153810,
+  "<audio_1739>": 153811,
+  "<audio_173>": 152245,
+  "<audio_1740>": 153812,
+  "<audio_1741>": 153813,
+  "<audio_1742>": 153814,
+  "<audio_1743>": 153815,
+  "<audio_1744>": 153816,
+  "<audio_1745>": 153817,
+  "<audio_1746>": 153818,
+  "<audio_1747>": 153819,
+  "<audio_1748>": 153820,
+  "<audio_1749>": 153821,
+  "<audio_174>": 152246,
+  "<audio_1750>": 153822,
+  "<audio_1751>": 153823,
+  "<audio_1752>": 153824,
+  "<audio_1753>": 153825,
+  "<audio_1754>": 153826,
+  "<audio_1755>": 153827,
+  "<audio_1756>": 153828,
+  "<audio_1757>": 153829,
+  "<audio_1758>": 153830,
+  "<audio_1759>": 153831,
+  "<audio_175>": 152247,
+  "<audio_1760>": 153832,
+  "<audio_1761>": 153833,
+  "<audio_1762>": 153834,
+  "<audio_1763>": 153835,
+  "<audio_1764>": 153836,
+  "<audio_1765>": 153837,
+  "<audio_1766>": 153838,
+  "<audio_1767>": 153839,
+  "<audio_1768>": 153840,
+  "<audio_1769>": 153841,
+  "<audio_176>": 152248,
+  "<audio_1770>": 153842,
+  "<audio_1771>": 153843,
+  "<audio_1772>": 153844,
+  "<audio_1773>": 153845,
+  "<audio_1774>": 153846,
+  "<audio_1775>": 153847,
+  "<audio_1776>": 153848,
+  "<audio_1777>": 153849,
+  "<audio_1778>": 153850,
+  "<audio_1779>": 153851,
+  "<audio_177>": 152249,
+  "<audio_1780>": 153852,
+  "<audio_1781>": 153853,
+  "<audio_1782>": 153854,
+  "<audio_1783>": 153855,
+  "<audio_1784>": 153856,
+  "<audio_1785>": 153857,
+  "<audio_1786>": 153858,
+  "<audio_1787>": 153859,
+  "<audio_1788>": 153860,
+  "<audio_1789>": 153861,
+  "<audio_178>": 152250,
+  "<audio_1790>": 153862,
+  "<audio_1791>": 153863,
+  "<audio_1792>": 153864,
+  "<audio_1793>": 153865,
+  "<audio_1794>": 153866,
+  "<audio_1795>": 153867,
+  "<audio_1796>": 153868,
+  "<audio_1797>": 153869,
+  "<audio_1798>": 153870,
+  "<audio_1799>": 153871,
+  "<audio_179>": 152251,
+  "<audio_17>": 152089,
+  "<audio_1800>": 153872,
+  "<audio_1801>": 153873,
+  "<audio_1802>": 153874,
+  "<audio_1803>": 153875,
+  "<audio_1804>": 153876,
+  "<audio_1805>": 153877,
+  "<audio_1806>": 153878,
+  "<audio_1807>": 153879,
+  "<audio_1808>": 153880,
+  "<audio_1809>": 153881,
+  "<audio_180>": 152252,
+  "<audio_1810>": 153882,
+  "<audio_1811>": 153883,
+  "<audio_1812>": 153884,
+  "<audio_1813>": 153885,
+  "<audio_1814>": 153886,
+  "<audio_1815>": 153887,
+  "<audio_1816>": 153888,
+  "<audio_1817>": 153889,
+  "<audio_1818>": 153890,
+  "<audio_1819>": 153891,
+  "<audio_181>": 152253,
+  "<audio_1820>": 153892,
+  "<audio_1821>": 153893,
+  "<audio_1822>": 153894,
+  "<audio_1823>": 153895,
+  "<audio_1824>": 153896,
+  "<audio_1825>": 153897,
+  "<audio_1826>": 153898,
+  "<audio_1827>": 153899,
+  "<audio_1828>": 153900,
+  "<audio_1829>": 153901,
+  "<audio_182>": 152254,
+  "<audio_1830>": 153902,
+  "<audio_1831>": 153903,
+  "<audio_1832>": 153904,
+  "<audio_1833>": 153905,
+  "<audio_1834>": 153906,
+  "<audio_1835>": 153907,
+  "<audio_1836>": 153908,
+  "<audio_1837>": 153909,
+  "<audio_1838>": 153910,
+  "<audio_1839>": 153911,
+  "<audio_183>": 152255,
+  "<audio_1840>": 153912,
+  "<audio_1841>": 153913,
+  "<audio_1842>": 153914,
+  "<audio_1843>": 153915,
+  "<audio_1844>": 153916,
+  "<audio_1845>": 153917,
+  "<audio_1846>": 153918,
+  "<audio_1847>": 153919,
+  "<audio_1848>": 153920,
+  "<audio_1849>": 153921,
+  "<audio_184>": 152256,
+  "<audio_1850>": 153922,
+  "<audio_1851>": 153923,
+  "<audio_1852>": 153924,
+  "<audio_1853>": 153925,
+  "<audio_1854>": 153926,
+  "<audio_1855>": 153927,
+  "<audio_1856>": 153928,
+  "<audio_1857>": 153929,
+  "<audio_1858>": 153930,
+  "<audio_1859>": 153931,
+  "<audio_185>": 152257,
+  "<audio_1860>": 153932,
+  "<audio_1861>": 153933,
+  "<audio_1862>": 153934,
+  "<audio_1863>": 153935,
+  "<audio_1864>": 153936,
+  "<audio_1865>": 153937,
+  "<audio_1866>": 153938,
+  "<audio_1867>": 153939,
+  "<audio_1868>": 153940,
+  "<audio_1869>": 153941,
+  "<audio_186>": 152258,
+  "<audio_1870>": 153942,
+  "<audio_1871>": 153943,
+  "<audio_1872>": 153944,
+  "<audio_1873>": 153945,
+  "<audio_1874>": 153946,
+  "<audio_1875>": 153947,
+  "<audio_1876>": 153948,
+  "<audio_1877>": 153949,
+  "<audio_1878>": 153950,
+  "<audio_1879>": 153951,
+  "<audio_187>": 152259,
+  "<audio_1880>": 153952,
+  "<audio_1881>": 153953,
+  "<audio_1882>": 153954,
+  "<audio_1883>": 153955,
+  "<audio_1884>": 153956,
+  "<audio_1885>": 153957,
+  "<audio_1886>": 153958,
+  "<audio_1887>": 153959,
+  "<audio_1888>": 153960,
+  "<audio_1889>": 153961,
+  "<audio_188>": 152260,
+  "<audio_1890>": 153962,
+  "<audio_1891>": 153963,
+  "<audio_1892>": 153964,
+  "<audio_1893>": 153965,
+  "<audio_1894>": 153966,
+  "<audio_1895>": 153967,
+  "<audio_1896>": 153968,
+  "<audio_1897>": 153969,
+  "<audio_1898>": 153970,
+  "<audio_1899>": 153971,
+  "<audio_189>": 152261,
+  "<audio_18>": 152090,
+  "<audio_1900>": 153972,
+  "<audio_1901>": 153973,
+  "<audio_1902>": 153974,
+  "<audio_1903>": 153975,
+  "<audio_1904>": 153976,
+  "<audio_1905>": 153977,
+  "<audio_1906>": 153978,
+  "<audio_1907>": 153979,
+  "<audio_1908>": 153980,
+  "<audio_1909>": 153981,
+  "<audio_190>": 152262,
+  "<audio_1910>": 153982,
+  "<audio_1911>": 153983,
+  "<audio_1912>": 153984,
+  "<audio_1913>": 153985,
+  "<audio_1914>": 153986,
+  "<audio_1915>": 153987,
+  "<audio_1916>": 153988,
+  "<audio_1917>": 153989,
+  "<audio_1918>": 153990,
+  "<audio_1919>": 153991,
+  "<audio_191>": 152263,
+  "<audio_1920>": 153992,
+  "<audio_1921>": 153993,
+  "<audio_1922>": 153994,
+  "<audio_1923>": 153995,
+  "<audio_1924>": 153996,
+  "<audio_1925>": 153997,
+  "<audio_1926>": 153998,
+  "<audio_1927>": 153999,
+  "<audio_1928>": 154000,
+  "<audio_1929>": 154001,
+  "<audio_192>": 152264,
+  "<audio_1930>": 154002,
+  "<audio_1931>": 154003,
+  "<audio_1932>": 154004,
+  "<audio_1933>": 154005,
+  "<audio_1934>": 154006,
+  "<audio_1935>": 154007,
+  "<audio_1936>": 154008,
+  "<audio_1937>": 154009,
+  "<audio_1938>": 154010,
+  "<audio_1939>": 154011,
+  "<audio_193>": 152265,
+  "<audio_1940>": 154012,
+  "<audio_1941>": 154013,
+  "<audio_1942>": 154014,
+  "<audio_1943>": 154015,
+  "<audio_1944>": 154016,
+  "<audio_1945>": 154017,
+  "<audio_1946>": 154018,
+  "<audio_1947>": 154019,
+  "<audio_1948>": 154020,
+  "<audio_1949>": 154021,
+  "<audio_194>": 152266,
+  "<audio_1950>": 154022,
+  "<audio_1951>": 154023,
+  "<audio_1952>": 154024,
+  "<audio_1953>": 154025,
+  "<audio_1954>": 154026,
+  "<audio_1955>": 154027,
+  "<audio_1956>": 154028,
+  "<audio_1957>": 154029,
+  "<audio_1958>": 154030,
+  "<audio_1959>": 154031,
+  "<audio_195>": 152267,
+  "<audio_1960>": 154032,
+  "<audio_1961>": 154033,
+  "<audio_1962>": 154034,
+  "<audio_1963>": 154035,
+  "<audio_1964>": 154036,
+  "<audio_1965>": 154037,
+  "<audio_1966>": 154038,
+  "<audio_1967>": 154039,
+  "<audio_1968>": 154040,
+  "<audio_1969>": 154041,
+  "<audio_196>": 152268,
+  "<audio_1970>": 154042,
+  "<audio_1971>": 154043,
+  "<audio_1972>": 154044,
+  "<audio_1973>": 154045,
+  "<audio_1974>": 154046,
+  "<audio_1975>": 154047,
+  "<audio_1976>": 154048,
+  "<audio_1977>": 154049,
+  "<audio_1978>": 154050,
+  "<audio_1979>": 154051,
+  "<audio_197>": 152269,
+  "<audio_1980>": 154052,
+  "<audio_1981>": 154053,
+  "<audio_1982>": 154054,
+  "<audio_1983>": 154055,
+  "<audio_1984>": 154056,
+  "<audio_1985>": 154057,
+  "<audio_1986>": 154058,
+  "<audio_1987>": 154059,
+  "<audio_1988>": 154060,
+  "<audio_1989>": 154061,
+  "<audio_198>": 152270,
+  "<audio_1990>": 154062,
+  "<audio_1991>": 154063,
+  "<audio_1992>": 154064,
+  "<audio_1993>": 154065,
+  "<audio_1994>": 154066,
+  "<audio_1995>": 154067,
+  "<audio_1996>": 154068,
+  "<audio_1997>": 154069,
+  "<audio_1998>": 154070,
+  "<audio_1999>": 154071,
+  "<audio_199>": 152271,
+  "<audio_19>": 152091,
+  "<audio_1>": 152073,
+  "<audio_2000>": 154072,
+  "<audio_2001>": 154073,
+  "<audio_2002>": 154074,
+  "<audio_2003>": 154075,
+  "<audio_2004>": 154076,
+  "<audio_2005>": 154077,
+  "<audio_2006>": 154078,
+  "<audio_2007>": 154079,
+  "<audio_2008>": 154080,
+  "<audio_2009>": 154081,
+  "<audio_200>": 152272,
+  "<audio_2010>": 154082,
+  "<audio_2011>": 154083,
+  "<audio_2012>": 154084,
+  "<audio_2013>": 154085,
+  "<audio_2014>": 154086,
+  "<audio_2015>": 154087,
+  "<audio_2016>": 154088,
+  "<audio_2017>": 154089,
+  "<audio_2018>": 154090,
+  "<audio_2019>": 154091,
+  "<audio_201>": 152273,
+  "<audio_2020>": 154092,
+  "<audio_2021>": 154093,
+  "<audio_2022>": 154094,
+  "<audio_2023>": 154095,
+  "<audio_2024>": 154096,
+  "<audio_2025>": 154097,
+  "<audio_2026>": 154098,
+  "<audio_2027>": 154099,
+  "<audio_2028>": 154100,
+  "<audio_2029>": 154101,
+  "<audio_202>": 152274,
+  "<audio_2030>": 154102,
+  "<audio_2031>": 154103,
+  "<audio_2032>": 154104,
+  "<audio_2033>": 154105,
+  "<audio_2034>": 154106,
+  "<audio_2035>": 154107,
+  "<audio_2036>": 154108,
+  "<audio_2037>": 154109,
+  "<audio_2038>": 154110,
+  "<audio_2039>": 154111,
+  "<audio_203>": 152275,
+  "<audio_2040>": 154112,
+  "<audio_2041>": 154113,
+  "<audio_2042>": 154114,
+  "<audio_2043>": 154115,
+  "<audio_2044>": 154116,
+  "<audio_2045>": 154117,
+  "<audio_2046>": 154118,
+  "<audio_2047>": 154119,
+  "<audio_204>": 152276,
+  "<audio_205>": 152277,
+  "<audio_206>": 152278,
+  "<audio_207>": 152279,
+  "<audio_208>": 152280,
+  "<audio_209>": 152281,
+  "<audio_20>": 152092,
+  "<audio_210>": 152282,
+  "<audio_211>": 152283,
+  "<audio_212>": 152284,
+  "<audio_213>": 152285,
+  "<audio_214>": 152286,
+  "<audio_215>": 152287,
+  "<audio_216>": 152288,
+  "<audio_217>": 152289,
+  "<audio_218>": 152290,
+  "<audio_219>": 152291,
+  "<audio_21>": 152093,
+  "<audio_220>": 152292,
+  "<audio_221>": 152293,
+  "<audio_222>": 152294,
+  "<audio_223>": 152295,
+  "<audio_224>": 152296,
+  "<audio_225>": 152297,
+  "<audio_226>": 152298,
+  "<audio_227>": 152299,
+  "<audio_228>": 152300,
+  "<audio_229>": 152301,
+  "<audio_22>": 152094,
+  "<audio_230>": 152302,
+  "<audio_231>": 152303,
+  "<audio_232>": 152304,
+  "<audio_233>": 152305,
+  "<audio_234>": 152306,
+  "<audio_235>": 152307,
+  "<audio_236>": 152308,
+  "<audio_237>": 152309,
+  "<audio_238>": 152310,
+  "<audio_239>": 152311,
+  "<audio_23>": 152095,
+  "<audio_240>": 152312,
+  "<audio_241>": 152313,
+  "<audio_242>": 152314,
+  "<audio_243>": 152315,
+  "<audio_244>": 152316,
+  "<audio_245>": 152317,
+  "<audio_246>": 152318,
+  "<audio_247>": 152319,
+  "<audio_248>": 152320,
+  "<audio_249>": 152321,
+  "<audio_24>": 152096,
+  "<audio_250>": 152322,
+  "<audio_251>": 152323,
+  "<audio_252>": 152324,
+  "<audio_253>": 152325,
+  "<audio_254>": 152326,
+  "<audio_255>": 152327,
+  "<audio_256>": 152328,
+  "<audio_257>": 152329,
+  "<audio_258>": 152330,
+  "<audio_259>": 152331,
+  "<audio_25>": 152097,
+  "<audio_260>": 152332,
+  "<audio_261>": 152333,
+  "<audio_262>": 152334,
+  "<audio_263>": 152335,
+  "<audio_264>": 152336,
+  "<audio_265>": 152337,
+  "<audio_266>": 152338,
+  "<audio_267>": 152339,
+  "<audio_268>": 152340,
+  "<audio_269>": 152341,
+  "<audio_26>": 152098,
+  "<audio_270>": 152342,
+  "<audio_271>": 152343,
+  "<audio_272>": 152344,
+  "<audio_273>": 152345,
+  "<audio_274>": 152346,
+  "<audio_275>": 152347,
+  "<audio_276>": 152348,
+  "<audio_277>": 152349,
+  "<audio_278>": 152350,
+  "<audio_279>": 152351,
+  "<audio_27>": 152099,
+  "<audio_280>": 152352,
+  "<audio_281>": 152353,
+  "<audio_282>": 152354,
+  "<audio_283>": 152355,
+  "<audio_284>": 152356,
+  "<audio_285>": 152357,
+  "<audio_286>": 152358,
+  "<audio_287>": 152359,
+  "<audio_288>": 152360,
+  "<audio_289>": 152361,
+  "<audio_28>": 152100,
+  "<audio_290>": 152362,
+  "<audio_291>": 152363,
+  "<audio_292>": 152364,
+  "<audio_293>": 152365,
+  "<audio_294>": 152366,
+  "<audio_295>": 152367,
+  "<audio_296>": 152368,
+  "<audio_297>": 152369,
+  "<audio_298>": 152370,
+  "<audio_299>": 152371,
+  "<audio_29>": 152101,
+  "<audio_2>": 152074,
+  "<audio_300>": 152372,
+  "<audio_301>": 152373,
+  "<audio_302>": 152374,
+  "<audio_303>": 152375,
+  "<audio_304>": 152376,
+  "<audio_305>": 152377,
+  "<audio_306>": 152378,
+  "<audio_307>": 152379,
+  "<audio_308>": 152380,
+  "<audio_309>": 152381,
+  "<audio_30>": 152102,
+  "<audio_310>": 152382,
+  "<audio_311>": 152383,
+  "<audio_312>": 152384,
+  "<audio_313>": 152385,
+  "<audio_314>": 152386,
+  "<audio_315>": 152387,
+  "<audio_316>": 152388,
+  "<audio_317>": 152389,
+  "<audio_318>": 152390,
+  "<audio_319>": 152391,
+  "<audio_31>": 152103,
+  "<audio_320>": 152392,
+  "<audio_321>": 152393,
+  "<audio_322>": 152394,
+  "<audio_323>": 152395,
+  "<audio_324>": 152396,
+  "<audio_325>": 152397,
+  "<audio_326>": 152398,
+  "<audio_327>": 152399,
+  "<audio_328>": 152400,
+  "<audio_329>": 152401,
+  "<audio_32>": 152104,
+  "<audio_330>": 152402,
+  "<audio_331>": 152403,
+  "<audio_332>": 152404,
+  "<audio_333>": 152405,
+  "<audio_334>": 152406,
+  "<audio_335>": 152407,
+  "<audio_336>": 152408,
+  "<audio_337>": 152409,
+  "<audio_338>": 152410,
+  "<audio_339>": 152411,
+  "<audio_33>": 152105,
+  "<audio_340>": 152412,
+  "<audio_341>": 152413,
+  "<audio_342>": 152414,
+  "<audio_343>": 152415,
+  "<audio_344>": 152416,
+  "<audio_345>": 152417,
+  "<audio_346>": 152418,
+  "<audio_347>": 152419,
+  "<audio_348>": 152420,
+  "<audio_349>": 152421,
+  "<audio_34>": 152106,
+  "<audio_350>": 152422,
+  "<audio_351>": 152423,
+  "<audio_352>": 152424,
+  "<audio_353>": 152425,
+  "<audio_354>": 152426,
+  "<audio_355>": 152427,
+  "<audio_356>": 152428,
+  "<audio_357>": 152429,
+  "<audio_358>": 152430,
+  "<audio_359>": 152431,
+  "<audio_35>": 152107,
+  "<audio_360>": 152432,
+  "<audio_361>": 152433,
+  "<audio_362>": 152434,
+  "<audio_363>": 152435,
+  "<audio_364>": 152436,
+  "<audio_365>": 152437,
+  "<audio_366>": 152438,
+  "<audio_367>": 152439,
+  "<audio_368>": 152440,
+  "<audio_369>": 152441,
+  "<audio_36>": 152108,
+  "<audio_370>": 152442,
+  "<audio_371>": 152443,
+  "<audio_372>": 152444,
+  "<audio_373>": 152445,
+  "<audio_374>": 152446,
+  "<audio_375>": 152447,
+  "<audio_376>": 152448,
+  "<audio_377>": 152449,
+  "<audio_378>": 152450,
+  "<audio_379>": 152451,
+  "<audio_37>": 152109,
+  "<audio_380>": 152452,
+  "<audio_381>": 152453,
+  "<audio_382>": 152454,
+  "<audio_383>": 152455,
+  "<audio_384>": 152456,
+  "<audio_385>": 152457,
+  "<audio_386>": 152458,
+  "<audio_387>": 152459,
+  "<audio_388>": 152460,
+  "<audio_389>": 152461,
+  "<audio_38>": 152110,
+  "<audio_390>": 152462,
+  "<audio_391>": 152463,
+  "<audio_392>": 152464,
+  "<audio_393>": 152465,
+  "<audio_394>": 152466,
+  "<audio_395>": 152467,
+  "<audio_396>": 152468,
+  "<audio_397>": 152469,
+  "<audio_398>": 152470,
+  "<audio_399>": 152471,
+  "<audio_39>": 152111,
+  "<audio_3>": 152075,
+  "<audio_400>": 152472,
+  "<audio_401>": 152473,
+  "<audio_402>": 152474,
+  "<audio_403>": 152475,
+  "<audio_404>": 152476,
+  "<audio_405>": 152477,
+  "<audio_406>": 152478,
+  "<audio_407>": 152479,
+  "<audio_408>": 152480,
+  "<audio_409>": 152481,
+  "<audio_40>": 152112,
+  "<audio_410>": 152482,
+  "<audio_411>": 152483,
+  "<audio_412>": 152484,
+  "<audio_413>": 152485,
+  "<audio_414>": 152486,
+  "<audio_415>": 152487,
+  "<audio_416>": 152488,
+  "<audio_417>": 152489,
+  "<audio_418>": 152490,
+  "<audio_419>": 152491,
+  "<audio_41>": 152113,
+  "<audio_420>": 152492,
+  "<audio_421>": 152493,
+  "<audio_422>": 152494,
+  "<audio_423>": 152495,
+  "<audio_424>": 152496,
+  "<audio_425>": 152497,
+  "<audio_426>": 152498,
+  "<audio_427>": 152499,
+  "<audio_428>": 152500,
+  "<audio_429>": 152501,
+  "<audio_42>": 152114,
+  "<audio_430>": 152502,
+  "<audio_431>": 152503,
+  "<audio_432>": 152504,
+  "<audio_433>": 152505,
+  "<audio_434>": 152506,
+  "<audio_435>": 152507,
+  "<audio_436>": 152508,
+  "<audio_437>": 152509,
+  "<audio_438>": 152510,
+  "<audio_439>": 152511,
+  "<audio_43>": 152115,
+  "<audio_440>": 152512,
+  "<audio_441>": 152513,
+  "<audio_442>": 152514,
+  "<audio_443>": 152515,
+  "<audio_444>": 152516,
+  "<audio_445>": 152517,
+  "<audio_446>": 152518,
+  "<audio_447>": 152519,
+  "<audio_448>": 152520,
+  "<audio_449>": 152521,
+  "<audio_44>": 152116,
+  "<audio_450>": 152522,
+  "<audio_451>": 152523,
+  "<audio_452>": 152524,
+  "<audio_453>": 152525,
+  "<audio_454>": 152526,
+  "<audio_455>": 152527,
+  "<audio_456>": 152528,
+  "<audio_457>": 152529,
+  "<audio_458>": 152530,
+  "<audio_459>": 152531,
+  "<audio_45>": 152117,
+  "<audio_460>": 152532,
+  "<audio_461>": 152533,
+  "<audio_462>": 152534,
+  "<audio_463>": 152535,
+  "<audio_464>": 152536,
+  "<audio_465>": 152537,
+  "<audio_466>": 152538,
+  "<audio_467>": 152539,
+  "<audio_468>": 152540,
+  "<audio_469>": 152541,
+  "<audio_46>": 152118,
+  "<audio_470>": 152542,
+  "<audio_471>": 152543,
+  "<audio_472>": 152544,
+  "<audio_473>": 152545,
+  "<audio_474>": 152546,
+  "<audio_475>": 152547,
+  "<audio_476>": 152548,
+  "<audio_477>": 152549,
+  "<audio_478>": 152550,
+  "<audio_479>": 152551,
+  "<audio_47>": 152119,
+  "<audio_480>": 152552,
+  "<audio_481>": 152553,
+  "<audio_482>": 152554,
+  "<audio_483>": 152555,
+  "<audio_484>": 152556,
+  "<audio_485>": 152557,
+  "<audio_486>": 152558,
+  "<audio_487>": 152559,
+  "<audio_488>": 152560,
+  "<audio_489>": 152561,
+  "<audio_48>": 152120,
+  "<audio_490>": 152562,
+  "<audio_491>": 152563,
+  "<audio_492>": 152564,
+  "<audio_493>": 152565,
+  "<audio_494>": 152566,
+  "<audio_495>": 152567,
+  "<audio_496>": 152568,
+  "<audio_497>": 152569,
+  "<audio_498>": 152570,
+  "<audio_499>": 152571,
+  "<audio_49>": 152121,
+  "<audio_4>": 152076,
+  "<audio_500>": 152572,
+  "<audio_501>": 152573,
+  "<audio_502>": 152574,
+  "<audio_503>": 152575,
+  "<audio_504>": 152576,
+  "<audio_505>": 152577,
+  "<audio_506>": 152578,
+  "<audio_507>": 152579,
+  "<audio_508>": 152580,
+  "<audio_509>": 152581,
+  "<audio_50>": 152122,
+  "<audio_510>": 152582,
+  "<audio_511>": 152583,
+  "<audio_512>": 152584,
+  "<audio_513>": 152585,
+  "<audio_514>": 152586,
+  "<audio_515>": 152587,
+  "<audio_516>": 152588,
+  "<audio_517>": 152589,
+  "<audio_518>": 152590,
+  "<audio_519>": 152591,
+  "<audio_51>": 152123,
+  "<audio_520>": 152592,
+  "<audio_521>": 152593,
+  "<audio_522>": 152594,
+  "<audio_523>": 152595,
+  "<audio_524>": 152596,
+  "<audio_525>": 152597,
+  "<audio_526>": 152598,
+  "<audio_527>": 152599,
+  "<audio_528>": 152600,
+  "<audio_529>": 152601,
+  "<audio_52>": 152124,
+  "<audio_530>": 152602,
+  "<audio_531>": 152603,
+  "<audio_532>": 152604,
+  "<audio_533>": 152605,
+  "<audio_534>": 152606,
+  "<audio_535>": 152607,
+  "<audio_536>": 152608,
+  "<audio_537>": 152609,
+  "<audio_538>": 152610,
+  "<audio_539>": 152611,
+  "<audio_53>": 152125,
+  "<audio_540>": 152612,
+  "<audio_541>": 152613,
+  "<audio_542>": 152614,
+  "<audio_543>": 152615,
+  "<audio_544>": 152616,
+  "<audio_545>": 152617,
+  "<audio_546>": 152618,
+  "<audio_547>": 152619,
+  "<audio_548>": 152620,
+  "<audio_549>": 152621,
+  "<audio_54>": 152126,
+  "<audio_550>": 152622,
+  "<audio_551>": 152623,
+  "<audio_552>": 152624,
+  "<audio_553>": 152625,
+  "<audio_554>": 152626,
+  "<audio_555>": 152627,
+  "<audio_556>": 152628,
+  "<audio_557>": 152629,
+  "<audio_558>": 152630,
+  "<audio_559>": 152631,
+  "<audio_55>": 152127,
+  "<audio_560>": 152632,
+  "<audio_561>": 152633,
+  "<audio_562>": 152634,
+  "<audio_563>": 152635,
+  "<audio_564>": 152636,
+  "<audio_565>": 152637,
+  "<audio_566>": 152638,
+  "<audio_567>": 152639,
+  "<audio_568>": 152640,
+  "<audio_569>": 152641,
+  "<audio_56>": 152128,
+  "<audio_570>": 152642,
+  "<audio_571>": 152643,
+  "<audio_572>": 152644,
+  "<audio_573>": 152645,
+  "<audio_574>": 152646,
+  "<audio_575>": 152647,
+  "<audio_576>": 152648,
+  "<audio_577>": 152649,
+  "<audio_578>": 152650,
+  "<audio_579>": 152651,
+  "<audio_57>": 152129,
+  "<audio_580>": 152652,
+  "<audio_581>": 152653,
+  "<audio_582>": 152654,
+  "<audio_583>": 152655,
+  "<audio_584>": 152656,
+  "<audio_585>": 152657,
+  "<audio_586>": 152658,
+  "<audio_587>": 152659,
+  "<audio_588>": 152660,
+  "<audio_589>": 152661,
+  "<audio_58>": 152130,
+  "<audio_590>": 152662,
+  "<audio_591>": 152663,
+  "<audio_592>": 152664,
+  "<audio_593>": 152665,
+  "<audio_594>": 152666,
+  "<audio_595>": 152667,
+  "<audio_596>": 152668,
+  "<audio_597>": 152669,
+  "<audio_598>": 152670,
+  "<audio_599>": 152671,
+  "<audio_59>": 152131,
+  "<audio_5>": 152077,
+  "<audio_600>": 152672,
+  "<audio_601>": 152673,
+  "<audio_602>": 152674,
+  "<audio_603>": 152675,
+  "<audio_604>": 152676,
+  "<audio_605>": 152677,
+  "<audio_606>": 152678,
+  "<audio_607>": 152679,
+  "<audio_608>": 152680,
+  "<audio_609>": 152681,
+  "<audio_60>": 152132,
+  "<audio_610>": 152682,
+  "<audio_611>": 152683,
+  "<audio_612>": 152684,
+  "<audio_613>": 152685,
+  "<audio_614>": 152686,
+  "<audio_615>": 152687,
+  "<audio_616>": 152688,
+  "<audio_617>": 152689,
+  "<audio_618>": 152690,
+  "<audio_619>": 152691,
+  "<audio_61>": 152133,
+  "<audio_620>": 152692,
+  "<audio_621>": 152693,
+  "<audio_622>": 152694,
+  "<audio_623>": 152695,
+  "<audio_624>": 152696,
+  "<audio_625>": 152697,
+  "<audio_626>": 152698,
+  "<audio_627>": 152699,
+  "<audio_628>": 152700,
+  "<audio_629>": 152701,
+  "<audio_62>": 152134,
+  "<audio_630>": 152702,
+  "<audio_631>": 152703,
+  "<audio_632>": 152704,
+  "<audio_633>": 152705,
+  "<audio_634>": 152706,
+  "<audio_635>": 152707,
+  "<audio_636>": 152708,
+  "<audio_637>": 152709,
+  "<audio_638>": 152710,
+  "<audio_639>": 152711,
+  "<audio_63>": 152135,
+  "<audio_640>": 152712,
+  "<audio_641>": 152713,
+  "<audio_642>": 152714,
+  "<audio_643>": 152715,
+  "<audio_644>": 152716,
+  "<audio_645>": 152717,
+  "<audio_646>": 152718,
+  "<audio_647>": 152719,
+  "<audio_648>": 152720,
+  "<audio_649>": 152721,
+  "<audio_64>": 152136,
+  "<audio_650>": 152722,
+  "<audio_651>": 152723,
+  "<audio_652>": 152724,
+  "<audio_653>": 152725,
+  "<audio_654>": 152726,
+  "<audio_655>": 152727,
+  "<audio_656>": 152728,
+  "<audio_657>": 152729,
+  "<audio_658>": 152730,
+  "<audio_659>": 152731,
+  "<audio_65>": 152137,
+  "<audio_660>": 152732,
+  "<audio_661>": 152733,
+  "<audio_662>": 152734,
+  "<audio_663>": 152735,
+  "<audio_664>": 152736,
+  "<audio_665>": 152737,
+  "<audio_666>": 152738,
+  "<audio_667>": 152739,
+  "<audio_668>": 152740,
+  "<audio_669>": 152741,
+  "<audio_66>": 152138,
+  "<audio_670>": 152742,
+  "<audio_671>": 152743,
+  "<audio_672>": 152744,
+  "<audio_673>": 152745,
+  "<audio_674>": 152746,
+  "<audio_675>": 152747,
+  "<audio_676>": 152748,
+  "<audio_677>": 152749,
+  "<audio_678>": 152750,
+  "<audio_679>": 152751,
+  "<audio_67>": 152139,
+  "<audio_680>": 152752,
+  "<audio_681>": 152753,
+  "<audio_682>": 152754,
+  "<audio_683>": 152755,
+  "<audio_684>": 152756,
+  "<audio_685>": 152757,
+  "<audio_686>": 152758,
+  "<audio_687>": 152759,
+  "<audio_688>": 152760,
+  "<audio_689>": 152761,
+  "<audio_68>": 152140,
+  "<audio_690>": 152762,
+  "<audio_691>": 152763,
+  "<audio_692>": 152764,
+  "<audio_693>": 152765,
+  "<audio_694>": 152766,
+  "<audio_695>": 152767,
+  "<audio_696>": 152768,
+  "<audio_697>": 152769,
+  "<audio_698>": 152770,
+  "<audio_699>": 152771,
+  "<audio_69>": 152141,
+  "<audio_6>": 152078,
+  "<audio_700>": 152772,
+  "<audio_701>": 152773,
+  "<audio_702>": 152774,
+  "<audio_703>": 152775,
+  "<audio_704>": 152776,
+  "<audio_705>": 152777,
+  "<audio_706>": 152778,
+  "<audio_707>": 152779,
+  "<audio_708>": 152780,
+  "<audio_709>": 152781,
+  "<audio_70>": 152142,
+  "<audio_710>": 152782,
+  "<audio_711>": 152783,
+  "<audio_712>": 152784,
+  "<audio_713>": 152785,
+  "<audio_714>": 152786,
+  "<audio_715>": 152787,
+  "<audio_716>": 152788,
+  "<audio_717>": 152789,
+  "<audio_718>": 152790,
+  "<audio_719>": 152791,
+  "<audio_71>": 152143,
+  "<audio_720>": 152792,
+  "<audio_721>": 152793,
+  "<audio_722>": 152794,
+  "<audio_723>": 152795,
+  "<audio_724>": 152796,
+  "<audio_725>": 152797,
+  "<audio_726>": 152798,
+  "<audio_727>": 152799,
+  "<audio_728>": 152800,
+  "<audio_729>": 152801,
+  "<audio_72>": 152144,
+  "<audio_730>": 152802,
+  "<audio_731>": 152803,
+  "<audio_732>": 152804,
+  "<audio_733>": 152805,
+  "<audio_734>": 152806,
+  "<audio_735>": 152807,
+  "<audio_736>": 152808,
+  "<audio_737>": 152809,
+  "<audio_738>": 152810,
+  "<audio_739>": 152811,
+  "<audio_73>": 152145,
+  "<audio_740>": 152812,
+  "<audio_741>": 152813,
+  "<audio_742>": 152814,
+  "<audio_743>": 152815,
+  "<audio_744>": 152816,
+  "<audio_745>": 152817,
+  "<audio_746>": 152818,
+  "<audio_747>": 152819,
+  "<audio_748>": 152820,
+  "<audio_749>": 152821,
+  "<audio_74>": 152146,
+  "<audio_750>": 152822,
+  "<audio_751>": 152823,
+  "<audio_752>": 152824,
+  "<audio_753>": 152825,
+  "<audio_754>": 152826,
+  "<audio_755>": 152827,
+  "<audio_756>": 152828,
+  "<audio_757>": 152829,
+  "<audio_758>": 152830,
+  "<audio_759>": 152831,
+  "<audio_75>": 152147,
+  "<audio_760>": 152832,
+  "<audio_761>": 152833,
+  "<audio_762>": 152834,
+  "<audio_763>": 152835,
+  "<audio_764>": 152836,
+  "<audio_765>": 152837,
+  "<audio_766>": 152838,
+  "<audio_767>": 152839,
+  "<audio_768>": 152840,
+  "<audio_769>": 152841,
+  "<audio_76>": 152148,
+  "<audio_770>": 152842,
+  "<audio_771>": 152843,
+  "<audio_772>": 152844,
+  "<audio_773>": 152845,
+  "<audio_774>": 152846,
+  "<audio_775>": 152847,
+  "<audio_776>": 152848,
+  "<audio_777>": 152849,
+  "<audio_778>": 152850,
+  "<audio_779>": 152851,
+  "<audio_77>": 152149,
+  "<audio_780>": 152852,
+  "<audio_781>": 152853,
+  "<audio_782>": 152854,
+  "<audio_783>": 152855,
+  "<audio_784>": 152856,
+  "<audio_785>": 152857,
+  "<audio_786>": 152858,
+  "<audio_787>": 152859,
+  "<audio_788>": 152860,
+  "<audio_789>": 152861,
+  "<audio_78>": 152150,
+  "<audio_790>": 152862,
+  "<audio_791>": 152863,
+  "<audio_792>": 152864,
+  "<audio_793>": 152865,
+  "<audio_794>": 152866,
+  "<audio_795>": 152867,
+  "<audio_796>": 152868,
+  "<audio_797>": 152869,
+  "<audio_798>": 152870,
+  "<audio_799>": 152871,
+  "<audio_79>": 152151,
+  "<audio_7>": 152079,
+  "<audio_800>": 152872,
+  "<audio_801>": 152873,
+  "<audio_802>": 152874,
+  "<audio_803>": 152875,
+  "<audio_804>": 152876,
+  "<audio_805>": 152877,
+  "<audio_806>": 152878,
+  "<audio_807>": 152879,
+  "<audio_808>": 152880,
+  "<audio_809>": 152881,
+  "<audio_80>": 152152,
+  "<audio_810>": 152882,
+  "<audio_811>": 152883,
+  "<audio_812>": 152884,
+  "<audio_813>": 152885,
+  "<audio_814>": 152886,
+  "<audio_815>": 152887,
+  "<audio_816>": 152888,
+  "<audio_817>": 152889,
+  "<audio_818>": 152890,
+  "<audio_819>": 152891,
+  "<audio_81>": 152153,
+  "<audio_820>": 152892,
+  "<audio_821>": 152893,
+  "<audio_822>": 152894,
+  "<audio_823>": 152895,
+  "<audio_824>": 152896,
+  "<audio_825>": 152897,
+  "<audio_826>": 152898,
+  "<audio_827>": 152899,
+  "<audio_828>": 152900,
+  "<audio_829>": 152901,
+  "<audio_82>": 152154,
+  "<audio_830>": 152902,
+  "<audio_831>": 152903,
+  "<audio_832>": 152904,
+  "<audio_833>": 152905,
+  "<audio_834>": 152906,
+  "<audio_835>": 152907,
+  "<audio_836>": 152908,
+  "<audio_837>": 152909,
+  "<audio_838>": 152910,
+  "<audio_839>": 152911,
+  "<audio_83>": 152155,
+  "<audio_840>": 152912,
+  "<audio_841>": 152913,
+  "<audio_842>": 152914,
+  "<audio_843>": 152915,
+  "<audio_844>": 152916,
+  "<audio_845>": 152917,
+  "<audio_846>": 152918,
+  "<audio_847>": 152919,
+  "<audio_848>": 152920,
+  "<audio_849>": 152921,
+  "<audio_84>": 152156,
+  "<audio_850>": 152922,
+  "<audio_851>": 152923,
+  "<audio_852>": 152924,
+  "<audio_853>": 152925,
+  "<audio_854>": 152926,
+  "<audio_855>": 152927,
+  "<audio_856>": 152928,
+  "<audio_857>": 152929,
+  "<audio_858>": 152930,
+  "<audio_859>": 152931,
+  "<audio_85>": 152157,
+  "<audio_860>": 152932,
+  "<audio_861>": 152933,
+  "<audio_862>": 152934,
+  "<audio_863>": 152935,
+  "<audio_864>": 152936,
+  "<audio_865>": 152937,
+  "<audio_866>": 152938,
+  "<audio_867>": 152939,
+  "<audio_868>": 152940,
+  "<audio_869>": 152941,
+  "<audio_86>": 152158,
+  "<audio_870>": 152942,
+  "<audio_871>": 152943,
+  "<audio_872>": 152944,
+  "<audio_873>": 152945,
+  "<audio_874>": 152946,
+  "<audio_875>": 152947,
+  "<audio_876>": 152948,
+  "<audio_877>": 152949,
+  "<audio_878>": 152950,
+  "<audio_879>": 152951,
+  "<audio_87>": 152159,
+  "<audio_880>": 152952,
+  "<audio_881>": 152953,
+  "<audio_882>": 152954,
+  "<audio_883>": 152955,
+  "<audio_884>": 152956,
+  "<audio_885>": 152957,
+  "<audio_886>": 152958,
+  "<audio_887>": 152959,
+  "<audio_888>": 152960,
+  "<audio_889>": 152961,
+  "<audio_88>": 152160,
+  "<audio_890>": 152962,
+  "<audio_891>": 152963,
+  "<audio_892>": 152964,
+  "<audio_893>": 152965,
+  "<audio_894>": 152966,
+  "<audio_895>": 152967,
+  "<audio_896>": 152968,
+  "<audio_897>": 152969,
+  "<audio_898>": 152970,
+  "<audio_899>": 152971,
+  "<audio_89>": 152161,
+  "<audio_8>": 152080,
+  "<audio_900>": 152972,
+  "<audio_901>": 152973,
+  "<audio_902>": 152974,
+  "<audio_903>": 152975,
+  "<audio_904>": 152976,
+  "<audio_905>": 152977,
+  "<audio_906>": 152978,
+  "<audio_907>": 152979,
+  "<audio_908>": 152980,
+  "<audio_909>": 152981,
+  "<audio_90>": 152162,
+  "<audio_910>": 152982,
+  "<audio_911>": 152983,
+  "<audio_912>": 152984,
+  "<audio_913>": 152985,
+  "<audio_914>": 152986,
+  "<audio_915>": 152987,
+  "<audio_916>": 152988,
+  "<audio_917>": 152989,
+  "<audio_918>": 152990,
+  "<audio_919>": 152991,
+  "<audio_91>": 152163,
+  "<audio_920>": 152992,
+  "<audio_921>": 152993,
+  "<audio_922>": 152994,
+  "<audio_923>": 152995,
+  "<audio_924>": 152996,
+  "<audio_925>": 152997,
+  "<audio_926>": 152998,
+  "<audio_927>": 152999,
+  "<audio_928>": 153000,
+  "<audio_929>": 153001,
+  "<audio_92>": 152164,
+  "<audio_930>": 153002,
+  "<audio_931>": 153003,
+  "<audio_932>": 153004,
+  "<audio_933>": 153005,
+  "<audio_934>": 153006,
+  "<audio_935>": 153007,
+  "<audio_936>": 153008,
+  "<audio_937>": 153009,
+  "<audio_938>": 153010,
+  "<audio_939>": 153011,
+  "<audio_93>": 152165,
+  "<audio_940>": 153012,
+  "<audio_941>": 153013,
+  "<audio_942>": 153014,
+  "<audio_943>": 153015,
+  "<audio_944>": 153016,
+  "<audio_945>": 153017,
+  "<audio_946>": 153018,
+  "<audio_947>": 153019,
+  "<audio_948>": 153020,
+  "<audio_949>": 153021,
+  "<audio_94>": 152166,
+  "<audio_950>": 153022,
+  "<audio_951>": 153023,
+  "<audio_952>": 153024,
+  "<audio_953>": 153025,
+  "<audio_954>": 153026,
+  "<audio_955>": 153027,
+  "<audio_956>": 153028,
+  "<audio_957>": 153029,
+  "<audio_958>": 153030,
+  "<audio_959>": 153031,
+  "<audio_95>": 152167,
+  "<audio_960>": 153032,
+  "<audio_961>": 153033,
+  "<audio_962>": 153034,
+  "<audio_963>": 153035,
+  "<audio_964>": 153036,
+  "<audio_965>": 153037,
+  "<audio_966>": 153038,
+  "<audio_967>": 153039,
+  "<audio_968>": 153040,
+  "<audio_969>": 153041,
+  "<audio_96>": 152168,
+  "<audio_970>": 153042,
+  "<audio_971>": 153043,
+  "<audio_972>": 153044,
+  "<audio_973>": 153045,
+  "<audio_974>": 153046,
+  "<audio_975>": 153047,
+  "<audio_976>": 153048,
+  "<audio_977>": 153049,
+  "<audio_978>": 153050,
+  "<audio_979>": 153051,
+  "<audio_97>": 152169,
+  "<audio_980>": 153052,
+  "<audio_981>": 153053,
+  "<audio_982>": 153054,
+  "<audio_983>": 153055,
+  "<audio_984>": 153056,
+  "<audio_985>": 153057,
+  "<audio_986>": 153058,
+  "<audio_987>": 153059,
+  "<audio_988>": 153060,
+  "<audio_989>": 153061,
+  "<audio_98>": 152170,
+  "<audio_990>": 153062,
+  "<audio_991>": 153063,
+  "<audio_992>": 153064,
+  "<audio_993>": 153065,
+  "<audio_994>": 153066,
+  "<audio_995>": 153067,
+  "<audio_996>": 153068,
+  "<audio_997>": 153069,
+  "<audio_998>": 153070,
+  "<audio_999>": 153071,
+  "<audio_99>": 152171,
+  "<audio_9>": 152081,
+  "<audio_pad>": 152071,
+  "<eoa>": 152070,
+  "<im_col>": 152067,
+  "<im_end>": 152065,
+  "<im_patch>": 152066,
+  "<im_start>": 152064,
+  "<soa>": 152069,
+  "<spk1>": 154120,
+  "<spk2>": 154121,
+  "<spk3>": 154122,
+  "<spk4>": 154123,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image|>": 152068,
+  "|<EXTRA_TOKENS_0>|": 151646,
+  "|<EXTRA_TOKENS_100>|": 151746,
+  "|<EXTRA_TOKENS_101>|": 151747,
+  "|<EXTRA_TOKENS_102>|": 151748,
+  "|<EXTRA_TOKENS_103>|": 151749,
+  "|<EXTRA_TOKENS_104>|": 151750,
+  "|<EXTRA_TOKENS_105>|": 151751,
+  "|<EXTRA_TOKENS_106>|": 151752,
+  "|<EXTRA_TOKENS_107>|": 151753,
+  "|<EXTRA_TOKENS_108>|": 151754,
+  "|<EXTRA_TOKENS_109>|": 151755,
+  "|<EXTRA_TOKENS_10>|": 151656,
+  "|<EXTRA_TOKENS_110>|": 151756,
+  "|<EXTRA_TOKENS_111>|": 151757,
+  "|<EXTRA_TOKENS_112>|": 151758,
+  "|<EXTRA_TOKENS_113>|": 151759,
+  "|<EXTRA_TOKENS_114>|": 151760,
+  "|<EXTRA_TOKENS_115>|": 151761,
+  "|<EXTRA_TOKENS_116>|": 151762,
+  "|<EXTRA_TOKENS_117>|": 151763,
+  "|<EXTRA_TOKENS_118>|": 151764,
+  "|<EXTRA_TOKENS_119>|": 151765,
+  "|<EXTRA_TOKENS_11>|": 151657,
+  "|<EXTRA_TOKENS_120>|": 151766,
+  "|<EXTRA_TOKENS_121>|": 151767,
+  "|<EXTRA_TOKENS_122>|": 151768,
+  "|<EXTRA_TOKENS_123>|": 151769,
+  "|<EXTRA_TOKENS_124>|": 151770,
+  "|<EXTRA_TOKENS_125>|": 151771,
+  "|<EXTRA_TOKENS_126>|": 151772,
+  "|<EXTRA_TOKENS_127>|": 151773,
+  "|<EXTRA_TOKENS_128>|": 151774,
+  "|<EXTRA_TOKENS_129>|": 151775,
+  "|<EXTRA_TOKENS_12>|": 151658,
+  "|<EXTRA_TOKENS_130>|": 151776,
+  "|<EXTRA_TOKENS_131>|": 151777,
+  "|<EXTRA_TOKENS_132>|": 151778,
+  "|<EXTRA_TOKENS_133>|": 151779,
+  "|<EXTRA_TOKENS_134>|": 151780,
+  "|<EXTRA_TOKENS_135>|": 151781,
+  "|<EXTRA_TOKENS_136>|": 151782,
+  "|<EXTRA_TOKENS_137>|": 151783,
+  "|<EXTRA_TOKENS_138>|": 151784,
+  "|<EXTRA_TOKENS_139>|": 151785,
+  "|<EXTRA_TOKENS_13>|": 151659,
+  "|<EXTRA_TOKENS_140>|": 151786,
+  "|<EXTRA_TOKENS_141>|": 151787,
+  "|<EXTRA_TOKENS_142>|": 151788,
+  "|<EXTRA_TOKENS_143>|": 151789,
+  "|<EXTRA_TOKENS_144>|": 151790,
+  "|<EXTRA_TOKENS_145>|": 151791,
+  "|<EXTRA_TOKENS_146>|": 151792,
+  "|<EXTRA_TOKENS_147>|": 151793,
+  "|<EXTRA_TOKENS_148>|": 151794,
+  "|<EXTRA_TOKENS_149>|": 151795,
+  "|<EXTRA_TOKENS_14>|": 151660,
+  "|<EXTRA_TOKENS_150>|": 151796,
+  "|<EXTRA_TOKENS_151>|": 151797,
+  "|<EXTRA_TOKENS_152>|": 151798,
+  "|<EXTRA_TOKENS_153>|": 151799,
+  "|<EXTRA_TOKENS_154>|": 151800,
+  "|<EXTRA_TOKENS_155>|": 151801,
+  "|<EXTRA_TOKENS_156>|": 151802,
+  "|<EXTRA_TOKENS_157>|": 151803,
+  "|<EXTRA_TOKENS_158>|": 151804,
+  "|<EXTRA_TOKENS_159>|": 151805,
+  "|<EXTRA_TOKENS_15>|": 151661,
+  "|<EXTRA_TOKENS_160>|": 151806,
+  "|<EXTRA_TOKENS_161>|": 151807,
+  "|<EXTRA_TOKENS_162>|": 151808,
+  "|<EXTRA_TOKENS_163>|": 151809,
+  "|<EXTRA_TOKENS_164>|": 151810,
+  "|<EXTRA_TOKENS_165>|": 151811,
+  "|<EXTRA_TOKENS_166>|": 151812,
+  "|<EXTRA_TOKENS_167>|": 151813,
+  "|<EXTRA_TOKENS_168>|": 151814,
+  "|<EXTRA_TOKENS_169>|": 151815,
+  "|<EXTRA_TOKENS_16>|": 151662,
+  "|<EXTRA_TOKENS_170>|": 151816,
+  "|<EXTRA_TOKENS_171>|": 151817,
+  "|<EXTRA_TOKENS_172>|": 151818,
+  "|<EXTRA_TOKENS_173>|": 151819,
+  "|<EXTRA_TOKENS_174>|": 151820,
+  "|<EXTRA_TOKENS_175>|": 151821,
+  "|<EXTRA_TOKENS_176>|": 151822,
+  "|<EXTRA_TOKENS_177>|": 151823,
+  "|<EXTRA_TOKENS_178>|": 151824,
+  "|<EXTRA_TOKENS_179>|": 151825,
+  "|<EXTRA_TOKENS_17>|": 151663,
+  "|<EXTRA_TOKENS_180>|": 151826,
+  "|<EXTRA_TOKENS_181>|": 151827,
+  "|<EXTRA_TOKENS_182>|": 151828,
+  "|<EXTRA_TOKENS_183>|": 151829,
+  "|<EXTRA_TOKENS_184>|": 151830,
+  "|<EXTRA_TOKENS_185>|": 151831,
+  "|<EXTRA_TOKENS_186>|": 151832,
+  "|<EXTRA_TOKENS_187>|": 151833,
+  "|<EXTRA_TOKENS_188>|": 151834,
+  "|<EXTRA_TOKENS_189>|": 151835,
+  "|<EXTRA_TOKENS_18>|": 151664,
+  "|<EXTRA_TOKENS_190>|": 151836,
+  "|<EXTRA_TOKENS_191>|": 151837,
+  "|<EXTRA_TOKENS_192>|": 151838,
+  "|<EXTRA_TOKENS_193>|": 151839,
+  "|<EXTRA_TOKENS_194>|": 151840,
+  "|<EXTRA_TOKENS_195>|": 151841,
+  "|<EXTRA_TOKENS_196>|": 151842,
+  "|<EXTRA_TOKENS_197>|": 151843,
+  "|<EXTRA_TOKENS_198>|": 151844,
+  "|<EXTRA_TOKENS_199>|": 151845,
+  "|<EXTRA_TOKENS_19>|": 151665,
+  "|<EXTRA_TOKENS_1>|": 151647,
+  "|<EXTRA_TOKENS_200>|": 151846,
+  "|<EXTRA_TOKENS_201>|": 151847,
+  "|<EXTRA_TOKENS_202>|": 151848,
+  "|<EXTRA_TOKENS_203>|": 151849,
+  "|<EXTRA_TOKENS_204>|": 151850,
+  "|<EXTRA_TOKENS_205>|": 151851,
+  "|<EXTRA_TOKENS_206>|": 151852,
+  "|<EXTRA_TOKENS_207>|": 151853,
+  "|<EXTRA_TOKENS_208>|": 151854,
+  "|<EXTRA_TOKENS_209>|": 151855,
+  "|<EXTRA_TOKENS_20>|": 151666,
+  "|<EXTRA_TOKENS_210>|": 151856,
+  "|<EXTRA_TOKENS_211>|": 151857,
+  "|<EXTRA_TOKENS_212>|": 151858,
+  "|<EXTRA_TOKENS_213>|": 151859,
+  "|<EXTRA_TOKENS_214>|": 151860,
+  "|<EXTRA_TOKENS_215>|": 151861,
+  "|<EXTRA_TOKENS_216>|": 151862,
+  "|<EXTRA_TOKENS_217>|": 151863,
+  "|<EXTRA_TOKENS_218>|": 151864,
+  "|<EXTRA_TOKENS_219>|": 151865,
+  "|<EXTRA_TOKENS_21>|": 151667,
+  "|<EXTRA_TOKENS_220>|": 151866,
+  "|<EXTRA_TOKENS_221>|": 151867,
+  "|<EXTRA_TOKENS_222>|": 151868,
+  "|<EXTRA_TOKENS_223>|": 151869,
+  "|<EXTRA_TOKENS_224>|": 151870,
+  "|<EXTRA_TOKENS_225>|": 151871,
+  "|<EXTRA_TOKENS_226>|": 151872,
+  "|<EXTRA_TOKENS_227>|": 151873,
+  "|<EXTRA_TOKENS_228>|": 151874,
+  "|<EXTRA_TOKENS_229>|": 151875,
+  "|<EXTRA_TOKENS_22>|": 151668,
+  "|<EXTRA_TOKENS_230>|": 151876,
+  "|<EXTRA_TOKENS_231>|": 151877,
+  "|<EXTRA_TOKENS_232>|": 151878,
+  "|<EXTRA_TOKENS_233>|": 151879,
+  "|<EXTRA_TOKENS_234>|": 151880,
+  "|<EXTRA_TOKENS_235>|": 151881,
+  "|<EXTRA_TOKENS_236>|": 151882,
+  "|<EXTRA_TOKENS_237>|": 151883,
+  "|<EXTRA_TOKENS_238>|": 151884,
+  "|<EXTRA_TOKENS_239>|": 151885,
+  "|<EXTRA_TOKENS_23>|": 151669,
+  "|<EXTRA_TOKENS_240>|": 151886,
+  "|<EXTRA_TOKENS_241>|": 151887,
+  "|<EXTRA_TOKENS_242>|": 151888,
+  "|<EXTRA_TOKENS_243>|": 151889,
+  "|<EXTRA_TOKENS_244>|": 151890,
+  "|<EXTRA_TOKENS_245>|": 151891,
+  "|<EXTRA_TOKENS_246>|": 151892,
+  "|<EXTRA_TOKENS_247>|": 151893,
+  "|<EXTRA_TOKENS_248>|": 151894,
+  "|<EXTRA_TOKENS_249>|": 151895,
+  "|<EXTRA_TOKENS_24>|": 151670,
+  "|<EXTRA_TOKENS_250>|": 151896,
+  "|<EXTRA_TOKENS_251>|": 151897,
+  "|<EXTRA_TOKENS_252>|": 151898,
+  "|<EXTRA_TOKENS_253>|": 151899,
+  "|<EXTRA_TOKENS_254>|": 151900,
+  "|<EXTRA_TOKENS_255>|": 151901,
+  "|<EXTRA_TOKENS_256>|": 151902,
+  "|<EXTRA_TOKENS_257>|": 151903,
+  "|<EXTRA_TOKENS_258>|": 151904,
+  "|<EXTRA_TOKENS_259>|": 151905,
+  "|<EXTRA_TOKENS_25>|": 151671,
+  "|<EXTRA_TOKENS_260>|": 151906,
+  "|<EXTRA_TOKENS_261>|": 151907,
+  "|<EXTRA_TOKENS_262>|": 151908,
+  "|<EXTRA_TOKENS_263>|": 151909,
+  "|<EXTRA_TOKENS_264>|": 151910,
+  "|<EXTRA_TOKENS_265>|": 151911,
+  "|<EXTRA_TOKENS_266>|": 151912,
+  "|<EXTRA_TOKENS_267>|": 151913,
+  "|<EXTRA_TOKENS_268>|": 151914,
+  "|<EXTRA_TOKENS_269>|": 151915,
+  "|<EXTRA_TOKENS_26>|": 151672,
+  "|<EXTRA_TOKENS_270>|": 151916,
+  "|<EXTRA_TOKENS_271>|": 151917,
+  "|<EXTRA_TOKENS_272>|": 151918,
+  "|<EXTRA_TOKENS_273>|": 151919,
+  "|<EXTRA_TOKENS_274>|": 151920,
+  "|<EXTRA_TOKENS_275>|": 151921,
+  "|<EXTRA_TOKENS_276>|": 151922,
+  "|<EXTRA_TOKENS_277>|": 151923,
+  "|<EXTRA_TOKENS_278>|": 151924,
+  "|<EXTRA_TOKENS_279>|": 151925,
+  "|<EXTRA_TOKENS_27>|": 151673,
+  "|<EXTRA_TOKENS_280>|": 151926,
+  "|<EXTRA_TOKENS_281>|": 151927,
+  "|<EXTRA_TOKENS_282>|": 151928,
+  "|<EXTRA_TOKENS_283>|": 151929,
+  "|<EXTRA_TOKENS_284>|": 151930,
+  "|<EXTRA_TOKENS_285>|": 151931,
+  "|<EXTRA_TOKENS_286>|": 151932,
+  "|<EXTRA_TOKENS_287>|": 151933,
+  "|<EXTRA_TOKENS_288>|": 151934,
+  "|<EXTRA_TOKENS_289>|": 151935,
+  "|<EXTRA_TOKENS_28>|": 151674,
+  "|<EXTRA_TOKENS_290>|": 151936,
+  "|<EXTRA_TOKENS_291>|": 151937,
+  "|<EXTRA_TOKENS_292>|": 151938,
+  "|<EXTRA_TOKENS_293>|": 151939,
+  "|<EXTRA_TOKENS_294>|": 151940,
+  "|<EXTRA_TOKENS_295>|": 151941,
+  "|<EXTRA_TOKENS_296>|": 151942,
+  "|<EXTRA_TOKENS_297>|": 151943,
+  "|<EXTRA_TOKENS_298>|": 151944,
+  "|<EXTRA_TOKENS_299>|": 151945,
+  "|<EXTRA_TOKENS_29>|": 151675,
+  "|<EXTRA_TOKENS_2>|": 151648,
+  "|<EXTRA_TOKENS_300>|": 151946,
+  "|<EXTRA_TOKENS_301>|": 151947,
+  "|<EXTRA_TOKENS_302>|": 151948,
+  "|<EXTRA_TOKENS_303>|": 151949,
+  "|<EXTRA_TOKENS_304>|": 151950,
+  "|<EXTRA_TOKENS_305>|": 151951,
+  "|<EXTRA_TOKENS_306>|": 151952,
+  "|<EXTRA_TOKENS_307>|": 151953,
+  "|<EXTRA_TOKENS_308>|": 151954,
+  "|<EXTRA_TOKENS_309>|": 151955,
+  "|<EXTRA_TOKENS_30>|": 151676,
+  "|<EXTRA_TOKENS_310>|": 151956,
+  "|<EXTRA_TOKENS_311>|": 151957,
+  "|<EXTRA_TOKENS_312>|": 151958,
+  "|<EXTRA_TOKENS_313>|": 151959,
+  "|<EXTRA_TOKENS_314>|": 151960,
+  "|<EXTRA_TOKENS_315>|": 151961,
+  "|<EXTRA_TOKENS_316>|": 151962,
+  "|<EXTRA_TOKENS_317>|": 151963,
+  "|<EXTRA_TOKENS_318>|": 151964,
+  "|<EXTRA_TOKENS_319>|": 151965,
+  "|<EXTRA_TOKENS_31>|": 151677,
+  "|<EXTRA_TOKENS_320>|": 151966,
+  "|<EXTRA_TOKENS_321>|": 151967,
+  "|<EXTRA_TOKENS_322>|": 151968,
+  "|<EXTRA_TOKENS_323>|": 151969,
+  "|<EXTRA_TOKENS_324>|": 151970,
+  "|<EXTRA_TOKENS_325>|": 151971,
+  "|<EXTRA_TOKENS_326>|": 151972,
+  "|<EXTRA_TOKENS_327>|": 151973,
+  "|<EXTRA_TOKENS_328>|": 151974,
+  "|<EXTRA_TOKENS_329>|": 151975,
+  "|<EXTRA_TOKENS_32>|": 151678,
+  "|<EXTRA_TOKENS_330>|": 151976,
+  "|<EXTRA_TOKENS_331>|": 151977,
+  "|<EXTRA_TOKENS_332>|": 151978,
+  "|<EXTRA_TOKENS_333>|": 151979,
+  "|<EXTRA_TOKENS_334>|": 151980,
+  "|<EXTRA_TOKENS_335>|": 151981,
+  "|<EXTRA_TOKENS_336>|": 151982,
+  "|<EXTRA_TOKENS_337>|": 151983,
+  "|<EXTRA_TOKENS_338>|": 151984,
+  "|<EXTRA_TOKENS_339>|": 151985,
+  "|<EXTRA_TOKENS_33>|": 151679,
+  "|<EXTRA_TOKENS_340>|": 151986,
+  "|<EXTRA_TOKENS_341>|": 151987,
+  "|<EXTRA_TOKENS_342>|": 151988,
+  "|<EXTRA_TOKENS_343>|": 151989,
+  "|<EXTRA_TOKENS_344>|": 151990,
+  "|<EXTRA_TOKENS_345>|": 151991,
+  "|<EXTRA_TOKENS_346>|": 151992,
+  "|<EXTRA_TOKENS_347>|": 151993,
+  "|<EXTRA_TOKENS_348>|": 151994,
+  "|<EXTRA_TOKENS_349>|": 151995,
+  "|<EXTRA_TOKENS_34>|": 151680,
+  "|<EXTRA_TOKENS_350>|": 151996,
+  "|<EXTRA_TOKENS_351>|": 151997,
+  "|<EXTRA_TOKENS_352>|": 151998,
+  "|<EXTRA_TOKENS_353>|": 151999,
+  "|<EXTRA_TOKENS_354>|": 152000,
+  "|<EXTRA_TOKENS_355>|": 152001,
+  "|<EXTRA_TOKENS_356>|": 152002,
+  "|<EXTRA_TOKENS_357>|": 152003,
+  "|<EXTRA_TOKENS_358>|": 152004,
+  "|<EXTRA_TOKENS_359>|": 152005,
+  "|<EXTRA_TOKENS_35>|": 151681,
+  "|<EXTRA_TOKENS_360>|": 152006,
+  "|<EXTRA_TOKENS_361>|": 152007,
+  "|<EXTRA_TOKENS_362>|": 152008,
+  "|<EXTRA_TOKENS_363>|": 152009,
+  "|<EXTRA_TOKENS_364>|": 152010,
+  "|<EXTRA_TOKENS_365>|": 152011,
+  "|<EXTRA_TOKENS_366>|": 152012,
+  "|<EXTRA_TOKENS_367>|": 152013,
+  "|<EXTRA_TOKENS_368>|": 152014,
+  "|<EXTRA_TOKENS_369>|": 152015,
+  "|<EXTRA_TOKENS_36>|": 151682,
+  "|<EXTRA_TOKENS_370>|": 152016,
+  "|<EXTRA_TOKENS_371>|": 152017,
+  "|<EXTRA_TOKENS_372>|": 152018,
+  "|<EXTRA_TOKENS_373>|": 152019,
+  "|<EXTRA_TOKENS_374>|": 152020,
+  "|<EXTRA_TOKENS_375>|": 152021,
+  "|<EXTRA_TOKENS_376>|": 152022,
+  "|<EXTRA_TOKENS_377>|": 152023,
+  "|<EXTRA_TOKENS_378>|": 152024,
+  "|<EXTRA_TOKENS_379>|": 152025,
+  "|<EXTRA_TOKENS_37>|": 151683,
+  "|<EXTRA_TOKENS_380>|": 152026,
+  "|<EXTRA_TOKENS_381>|": 152027,
+  "|<EXTRA_TOKENS_382>|": 152028,
+  "|<EXTRA_TOKENS_383>|": 152029,
+  "|<EXTRA_TOKENS_384>|": 152030,
+  "|<EXTRA_TOKENS_385>|": 152031,
+  "|<EXTRA_TOKENS_386>|": 152032,
+  "|<EXTRA_TOKENS_387>|": 152033,
+  "|<EXTRA_TOKENS_388>|": 152034,
+  "|<EXTRA_TOKENS_389>|": 152035,
+  "|<EXTRA_TOKENS_38>|": 151684,
+  "|<EXTRA_TOKENS_390>|": 152036,
+  "|<EXTRA_TOKENS_391>|": 152037,
+  "|<EXTRA_TOKENS_392>|": 152038,
+  "|<EXTRA_TOKENS_393>|": 152039,
+  "|<EXTRA_TOKENS_394>|": 152040,
+  "|<EXTRA_TOKENS_395>|": 152041,
+  "|<EXTRA_TOKENS_396>|": 152042,
+  "|<EXTRA_TOKENS_397>|": 152043,
+  "|<EXTRA_TOKENS_398>|": 152044,
+  "|<EXTRA_TOKENS_399>|": 152045,
+  "|<EXTRA_TOKENS_39>|": 151685,
+  "|<EXTRA_TOKENS_3>|": 151649,
+  "|<EXTRA_TOKENS_400>|": 152046,
+  "|<EXTRA_TOKENS_401>|": 152047,
+  "|<EXTRA_TOKENS_402>|": 152048,
+  "|<EXTRA_TOKENS_403>|": 152049,
+  "|<EXTRA_TOKENS_404>|": 152050,
+  "|<EXTRA_TOKENS_405>|": 152051,
+  "|<EXTRA_TOKENS_406>|": 152052,
+  "|<EXTRA_TOKENS_407>|": 152053,
+  "|<EXTRA_TOKENS_408>|": 152054,
+  "|<EXTRA_TOKENS_409>|": 152055,
+  "|<EXTRA_TOKENS_40>|": 151686,
+  "|<EXTRA_TOKENS_410>|": 152056,
+  "|<EXTRA_TOKENS_411>|": 152057,
+  "|<EXTRA_TOKENS_412>|": 152058,
+  "|<EXTRA_TOKENS_413>|": 152059,
+  "|<EXTRA_TOKENS_414>|": 152060,
+  "|<EXTRA_TOKENS_415>|": 152061,
+  "|<EXTRA_TOKENS_416>|": 152062,
+  "|<EXTRA_TOKENS_417>|": 152063,
+  "|<EXTRA_TOKENS_41>|": 151687,
+  "|<EXTRA_TOKENS_42>|": 151688,
+  "|<EXTRA_TOKENS_43>|": 151689,
+  "|<EXTRA_TOKENS_44>|": 151690,
+  "|<EXTRA_TOKENS_45>|": 151691,
+  "|<EXTRA_TOKENS_46>|": 151692,
+  "|<EXTRA_TOKENS_47>|": 151693,
+  "|<EXTRA_TOKENS_48>|": 151694,
+  "|<EXTRA_TOKENS_49>|": 151695,
+  "|<EXTRA_TOKENS_4>|": 151650,
+  "|<EXTRA_TOKENS_50>|": 151696,
+  "|<EXTRA_TOKENS_51>|": 151697,
+  "|<EXTRA_TOKENS_52>|": 151698,
+  "|<EXTRA_TOKENS_53>|": 151699,
+  "|<EXTRA_TOKENS_54>|": 151700,
+  "|<EXTRA_TOKENS_55>|": 151701,
+  "|<EXTRA_TOKENS_56>|": 151702,
+  "|<EXTRA_TOKENS_57>|": 151703,
+  "|<EXTRA_TOKENS_58>|": 151704,
+  "|<EXTRA_TOKENS_59>|": 151705,
+  "|<EXTRA_TOKENS_5>|": 151651,
+  "|<EXTRA_TOKENS_60>|": 151706,
+  "|<EXTRA_TOKENS_61>|": 151707,
+  "|<EXTRA_TOKENS_62>|": 151708,
+  "|<EXTRA_TOKENS_63>|": 151709,
+  "|<EXTRA_TOKENS_64>|": 151710,
+  "|<EXTRA_TOKENS_65>|": 151711,
+  "|<EXTRA_TOKENS_66>|": 151712,
+  "|<EXTRA_TOKENS_67>|": 151713,
+  "|<EXTRA_TOKENS_68>|": 151714,
+  "|<EXTRA_TOKENS_69>|": 151715,
+  "|<EXTRA_TOKENS_6>|": 151652,
+  "|<EXTRA_TOKENS_70>|": 151716,
+  "|<EXTRA_TOKENS_71>|": 151717,
+  "|<EXTRA_TOKENS_72>|": 151718,
+  "|<EXTRA_TOKENS_73>|": 151719,
+  "|<EXTRA_TOKENS_74>|": 151720,
+  "|<EXTRA_TOKENS_75>|": 151721,
+  "|<EXTRA_TOKENS_76>|": 151722,
+  "|<EXTRA_TOKENS_77>|": 151723,
+  "|<EXTRA_TOKENS_78>|": 151724,
+  "|<EXTRA_TOKENS_79>|": 151725,
+  "|<EXTRA_TOKENS_7>|": 151653,
+  "|<EXTRA_TOKENS_80>|": 151726,
+  "|<EXTRA_TOKENS_81>|": 151727,
+  "|<EXTRA_TOKENS_82>|": 151728,
+  "|<EXTRA_TOKENS_83>|": 151729,
+  "|<EXTRA_TOKENS_84>|": 151730,
+  "|<EXTRA_TOKENS_85>|": 151731,
+  "|<EXTRA_TOKENS_86>|": 151732,
+  "|<EXTRA_TOKENS_87>|": 151733,
+  "|<EXTRA_TOKENS_88>|": 151734,
+  "|<EXTRA_TOKENS_89>|": 151735,
+  "|<EXTRA_TOKENS_8>|": 151654,
+  "|<EXTRA_TOKENS_90>|": 151736,
+  "|<EXTRA_TOKENS_91>|": 151737,
+  "|<EXTRA_TOKENS_92>|": 151738,
+  "|<EXTRA_TOKENS_93>|": 151739,
+  "|<EXTRA_TOKENS_94>|": 151740,
+  "|<EXTRA_TOKENS_95>|": 151741,
+  "|<EXTRA_TOKENS_96>|": 151742,
+  "|<EXTRA_TOKENS_97>|": 151743,
+  "|<EXTRA_TOKENS_98>|": 151744,
+  "|<EXTRA_TOKENS_99>|": 151745,
+  "|<EXTRA_TOKENS_9>|": 151655
+}

audio_modules.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:565a255e42f1a7c8ebaf1179d94c94048db934b7b2bdf40c18ff0bfe64cb06ef
+size 532905901

chat_template.jinja ADDED Viewed

	@@ -0,0 +1,13 @@

+{% for message in messages -%}
+        {%- if (loop.index % 2 == 1 and message['role'] != 'user') or
+          (loop.index % 2 == 0 and message['role'].lower() != 'assistant') -%}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+        {%- endif -%}
+        {{ message['role'].capitalize() + ': ' + message['content'] }}
+        {%- if not loop.last -%}
+        {{ ' ' }}
+        {%- endif %}
+        {%- endfor -%}
+        {%- if add_generation_prompt -%}
+        {{ ' Assistant:' }}
+        {%- endif %}

config.json ADDED Viewed

	@@ -0,0 +1,44 @@

+{
+  "architectures": [
+    "MolmoAudioForCausalLM"
+  ],
+  "attention_layer_norm": false,
+  "audio_codebook_offset": 152072,
+  "audio_codebook_size": 2048,
+  "audio_max_input_tokens": 1024,
+  "audio_pad_token_id": 152071,
+  "auto_map": {
+    "AutoConfig": "config_molmo.MolmoConfig",
+    "AutoModelForCausalLM": "modeling_molmo.MolmoForCausalLM"
+  },
+  "clip_qkv": null,
+  "dtype": "bfloat16",
+  "embedding_size": 154124,
+  "eoa_token_id": 152070,
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 37888,
+  "layer_norm_eps": 1e-06,
+  "layer_norm_type": "rms",
+  "max_position_embeddings": 4096,
+  "model_type": "molmo",
+  "norm_after": false,
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "qkv_bias": true,
+  "rope_theta": 1000000.0,
+  "soa_token_id": 152069,
+  "spk_token_ids": [
+    154120,
+    154121,
+    154122,
+    154123
+  ],
+  "tie_word_embeddings": false,
+  "transformers_version": "4.43.3",
+  "use_cache": true,
+  "use_position_ids": true,
+  "vocab_size": 154124,
+  "weight_tying": false
+}

config_molmo.py ADDED Viewed

	@@ -0,0 +1,60 @@

+from typing import List
+from transformers import PretrainedConfig, AutoTokenizer
+class MolmoConfig(PretrainedConfig):
+    model_type = "molmo"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    def __init__(
+        self,
+        vocab_size=50304,
+        embedding_size=50304,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        use_cache=True,
+        layer_norm_eps: float = 1e-5,
+        rope_theta=10000.0,
+        clip_qkv=None,
+        qkv_bias: bool = False,
+        weight_tying: bool = False,
+        use_position_ids: bool=True,
+        tie_word_embeddings: bool=True,
+        attention_layer_norm: bool=False,
+        norm_after: bool = False,
+        layer_norm_type: str="rms",
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.embedding_size = embedding_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.layer_norm_eps = layer_norm_eps
+        self.weight_tying = weight_tying
+        self.use_position_ids = use_position_ids
+        self.attention_layer_norm = attention_layer_norm
+        self.num_key_value_heads = num_key_value_heads
+        self.initializer_range = initializer_range
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.clip_qkv = clip_qkv
+        self.qkv_bias = qkv_bias
+        self.norm_after = norm_after
+        self.tie_word_embeddings = tie_word_embeddings
+        self.layer_norm_type = layer_norm_type
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+MolmoConfig.register_for_auto_class()

generation_config.json ADDED Viewed

	@@ -0,0 +1,4 @@

+{
+  "_from_model_config": true,
+  "transformers_version": "4.43.3"
+}

image_preprocessing_molmo.py ADDED Viewed

	@@ -0,0 +1,547 @@

+"""Image processor class for Molmo"""
+from typing import List, Optional, Union, Mapping
+import numpy as np
+import einops
+import torch
+import torchvision.transforms
+from torchvision.transforms import InterpolationMode
+from torchvision.transforms.functional import convert_image_dtype
+from transformers.image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ImageInput,
+    is_valid_image,
+)
+from transformers.processing_utils import ImagesKwargs
+from transformers.image_processing_utils import BaseImageProcessor
+from transformers.utils import logging
+logger = logging.get_logger(__name__)
+def pad_to_bounding_box(
+    image, offset_height, offset_width, target_height,
+    target_width, value=0
+):
+    height, width = image.shape[:2]
+    after_padding_width = target_width - offset_width - width
+    after_padding_height = target_height - offset_height - height
+    return np.pad(image, [
+        [offset_height, after_padding_height],
+        [offset_width, after_padding_width],
+        [0, 0]
+    ], constant_values=value)
+def normalize_image(image, offset, scale):
+    image -= np.array(offset, dtype=np.float32)[None, None, :]
+    image /= np.array(scale, dtype=np.float32)[None, None, :]
+    return image
+def resize_and_pad(
+    image,
+    desired_output_size,
+    resize_method="torch-bilinear",
+    pad_value=0,
+    normalize=True,
+    image_mean=OPENAI_CLIP_MEAN,
+    image_std=OPENAI_CLIP_STD,
+):
+    desired_height, desired_width = desired_output_size
+    height, width = image.shape[:2]
+    # Cast into float32 since the training code did this in float32 and it (very rarely) effects
+    # the results after rounding.
+    image_scale_y = np.array(desired_height, np.float32) / np.array(height, np.float32)
+    image_scale_x = np.array(desired_width, np.float32) / np.array(width, np.float32)
+    image_scale = min(image_scale_x, image_scale_y)
+    scaled_height = int(np.array(height, np.float32) * image_scale)
+    scaled_width = int(np.array(width, np.float32) * image_scale)
+    if resize_method == "tensorflow":
+        # This how the original training code did resizing, it can produce slightly different
+        # results then using torch resize so we keep it just in case
+        import importlib
+        tf = importlib.import_module("tensorflow")
+        image = tf.image.convert_image_dtype(tf.constant(image), dtype=tf.float32)
+        image = tf.image.resize(
+            image,
+            [scaled_height, scaled_width],
+            method=tf.image.ResizeMethod.BILINEAR,
+            antialias=True,
+        )
+        image = tf.clip_by_value(image, 0.0, 1.0)
+        image = image.numpy()
+    elif resize_method == "torch-bilinear":
+        image = torch.permute(torch.from_numpy(image), [2, 0, 1])
+        image = convert_image_dtype(image)  # resize in float32 to match the training code
+        image = torchvision.transforms.Resize(
+            [scaled_height, scaled_width], InterpolationMode.BILINEAR, antialias=True
+        )(image)
+        image = torch.clip(image, 0.0, 1.0)
+        image = torch.permute(image, [1, 2, 0]).numpy()
+    else:
+        raise NotImplementedError(resize_method)
+    top_pad = (desired_height - scaled_height) // 2
+    left_pad = (desired_width - scaled_width) // 2
+    padding = [
+        [top_pad, desired_height - scaled_height - top_pad],
+        [left_pad, desired_width - scaled_width - left_pad],
+        [0, 0]
+    ]
+    image_mask = np.pad(np.ones_like(image[:, :, 0], dtype=bool), padding[:2])
+    image = np.pad(image, padding, constant_values=pad_value)
+    if normalize:
+        image = normalize_image(image, offset=image_mean, scale=image_std)
+    return image, image_mask
+def select_tiling(h, w, patch_size, max_num_patches):
+    """Decide how best to divide in image of size [w, h] in up to max_num_patches of size patch_size"""
+    original_size = np.stack([h, w])  # [1, 2]
+    original_res = h * w
+    tilings = []
+    for i in range(1, max_num_patches+1):
+        for j in range(1, max_num_patches+1):
+            if i*j <= max_num_patches:
+                tilings.append((i, j))
+    # sort so argmin and argmax favour smaller tilings in the event of a tie
+    tilings.sort(key=lambda x: (x[0]*x[1], x[0]))
+    candidate_tilings = np.array(tilings, dtype=np.int32)  # [n_resolutions, 2]
+    candidate_resolutions = candidate_tilings * patch_size  # [n_resolutions, 2]
+    # How much we would need to scale the image to fit exactly in each tiling
+    original_size = np.stack([h, w], dtype=np.float32)  # [1, 2]
+    required_scale_d = candidate_resolutions.astype(np.float32) / original_size
+    required_scale = np.min(required_scale_d, axis=-1, keepdims=True)  # [n_resolutions, 1]
+    if np.all(required_scale < 1):
+        # We are forced to downscale, so try to minimize the amount of downscaling
+        ix = np.argmax(required_scale)
+    else:
+        # Pick the resolution that required the least upscaling so that it most closely fits the image
+        required_scale = np.where(required_scale < 1.0, 10e9, required_scale)
+        ix = np.argmin(required_scale)
+    return candidate_tilings[ix]
+class MolmoImagesKwargs(ImagesKwargs, total=False):
+    max_crops: Optional[int]
+    overlap_margins: Optional[List[int]]
+    base_image_input_size: Optional[List[int]]
+    image_token_length_w: Optional[int]
+    image_token_length_h: Optional[int]
+    image_patch_size: Optional[int]
+    image_padding_mask: Optional[bool]
+class MolmoImageProcessor(BaseImageProcessor):
+    """Preprocess images and multi-model inputs"""
+    def __init__(
+        self,
+        max_crops: int = 12,
+        overlap_margins: List[int] = (4, 4),
+        base_image_input_size: List[int] = (336, 336),
+        image_token_length_w: int = 12,
+        image_token_length_h: int = 12,
+        image_patch_size: int = 14,
+        image_padding_mask: bool = True,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.max_crops = max_crops
+        self.overlap_margins = overlap_margins
+        self.base_image_input_size = base_image_input_size
+        self.image_token_length_w = image_token_length_w
+        self.image_token_length_h = image_token_length_h
+        self.image_patch_size = image_patch_size
+        self.image_padding_mask = image_padding_mask
+        self.do_normalize = do_normalize
+        self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+        self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+    def image_to_patches_and_tokens(
+        self,
+        image: ImageInput,
+        image_patch_token_id: int,
+        image_col_token_id: int,
+        image_start_token_id: int,
+        image_end_token_id: int,
+        max_crops: Optional[int] = None,
+        overlap_margins: Optional[List[int]] = None,
+        base_image_input_size: Optional[Union[int, List[int]]] = None,
+        image_token_length_w: Optional[int] = None,
+        image_token_length_h: Optional[int] = None,
+        image_patch_size: Optional[int] = None,
+    ):
+        if isinstance(base_image_input_size, int):
+            base_image_input_size = (base_image_input_size, base_image_input_size)
+        base_image_input_d = image_patch_size
+        tokens_per_image = image_token_length_w * image_token_length_h
+        image_base_patch_w = base_image_input_size[1] // base_image_input_d
+        image_base_patch_h = base_image_input_size[0] // base_image_input_d
+        original_image_h, original_image_w = image.shape[:2]
+        crop_size = base_image_input_size[0]
+        # Discard this many patches from the (left/top, right/bottom) of crops
+        left_margin, right_margin = overlap_margins
+        # left_margin, right_margin = 2, 2
+        assert left_margin % 2 == 0  # Required for compatibility with 2x2 pooling
+        total_margin_pixels = base_image_input_d*(right_margin + left_margin)  # pixels removed per dim
+        crop_patches = base_image_input_size[0] // base_image_input_d  # patches per crop dim
+        crop_window_patches = crop_patches - (right_margin + left_margin)  # usable patches
+        crop_window_size = crop_window_patches * base_image_input_d
+        tiling = select_tiling(
+            original_image_h - total_margin_pixels,
+            original_image_w - total_margin_pixels,
+            crop_window_size,
+            max_crops
+        )
+        src, img_mask = resize_and_pad(
+            image,
+            [tiling[0]*crop_window_size+total_margin_pixels, tiling[1]*crop_window_size+total_margin_pixels]
+        )
+        # Now we have to split the image into crops, while keeping track of how each patch in the
+        # each crop should be ordered in the global image, this require a lot of tricky booking
+        n_crops = tiling[0] * tiling[1]
+        patches_arr = []
+        mask_arr = []
+        patch_ordering_arr = []
+        # We assume 2x2 pooling, but can allow padding the right/bottom with extra
+        # patches if the number of patches per side is not even
+        assert (crop_patches+1)//2 == image_token_length_h
+        assert (crop_patches+1)//2 == image_token_length_w
+        on = 0
+        on_patch = 0
+        for i in range(tiling[0]):
+            y0 = i*crop_window_size
+            if i == 0:
+                crop_y0 = 0
+            else:
+                crop_y0 = left_margin // 2
+            crop_h = image_base_patch_h - (right_margin + left_margin)
+            if i == 0:
+                crop_h += left_margin
+            if i == (tiling[0]-1):
+                crop_h += right_margin
+            for j in range(tiling[1]):
+                x0 = j*crop_window_size
+                if j == 0:
+                    crop_x0 = 0
+                else:
+                    crop_x0 = left_margin // 2
+                crop_w = image_base_patch_w - (right_margin + left_margin)
+                if j == 0:
+                    crop_w += left_margin
+                if j == (tiling[1]-1):
+                    crop_w += right_margin
+                pooled_w = (crop_w + 1) // 2
+                pooled_h = (crop_h + 1) // 2
+                patch_ordering_arr.append(
+                    pad_to_bounding_box(
+                        np.reshape(np.arange(on, on+pooled_h*pooled_w, dtype=np.int32), (pooled_h, pooled_w, 1)),
+                        crop_y0, crop_x0, image_token_length_h, image_token_length_w, value=-1
+                    )[:, :, 0]
+                )
+                patches_arr.append(src[y0:y0+crop_size, x0:x0+crop_size])
+                mask_arr.append(img_mask[y0:y0+crop_size, x0:x0+crop_size])
+                on += pooled_h*pooled_w
+                on_patch += 1
+        patches = np.stack(patches_arr)
+        patch_ordering = np.stack(patch_ordering_arr)
+        img_mask = np.stack(mask_arr)
+        # Switch to [n_crops, n_patches, pixels_per_patch] format
+        image_layout_impatch_w, image_layout_impatch_h = tiling[0], tiling[1]
+        patches = einops.rearrange(
+            patches, 'p (h dh) (w dw) c -> p (h w) (dh dw c)',
+            dh=base_image_input_d,
+            dw=base_image_input_d,
+            h=image_base_patch_h,
+            w=image_base_patch_w
+        )
+        img_mask = einops.rearrange(
+            img_mask, 'p (h dh) (w dw) -> p (h w) (dh dw)',
+            dh=base_image_input_d,
+            dw=base_image_input_d,
+            h=image_base_patch_h,
+            w=image_base_patch_w
+        )
+        img_mask = img_mask.astype(np.float32).mean(axis=-1)
+        patch_ordering = np.reshape(patch_ordering, [-1])
+        valid = patch_ordering >= 0
+        # Transpose order, to get left-to-right order instead of crop-by-crop order
+        patch_ordering_rh = np.reshape(
+            patch_ordering,
+            [tiling[0], tiling[1], image_token_length_h, image_token_length_w]
+        )
+        patch_ordering_rh = np.transpose(patch_ordering_rh, [0, 2, 1, 3])
+        patch_ordering_rh = np.reshape(patch_ordering_rh, [-1])
+        # The transpose will screw up which patches are masked, project the
+        # new order into sparse structure of `patch_ordering` to fix this
+        patch_ordering[valid] = patch_ordering_rh[patch_ordering_rh >= 0]
+        # Now build the output tokens
+        h = tiling[0] * crop_window_patches + (right_margin+left_margin)
+        w = tiling[1] * crop_window_patches + (right_margin+left_margin)
+        per_row = np.full(
+            ((w+1)//2,),
+            image_patch_token_id,
+        )
+        per_row = np.concatenate([per_row, [image_col_token_id]], 0)
+        joint = np.tile(per_row, [(h+1)//2])
+        joint = [
+            [image_start_token_id],
+            joint,
+            [image_end_token_id]
+        ]
+        # Finally do the same for the global image
+        resized, _ = resize_and_pad(image, base_image_input_size)
+        resized = einops.rearrange(
+            resized, '(h dh) (w dw) c -> (h w) (dh dw c)',
+            dh=base_image_input_d,
+            dw=base_image_input_d,
+            h=image_base_patch_h,
+            w=image_base_patch_w
+        )
+        patches = np.concatenate([np.expand_dims(resized, 0), patches], 0)
+        # Global image goes first, so the order of patches in previous crops gets increased
+        patch_ordering = np.where(
+            patch_ordering >= 0,
+            patch_ordering + tokens_per_image,
+            -1
+        )
+        patch_ordering = np.concatenate([np.arange(0, tokens_per_image), patch_ordering], 0)
+        per_row = np.full(
+            (image_token_length_w,),
+            image_patch_token_id,
+        )
+        per_row = np.concatenate([per_row, [image_col_token_id]], 0)
+        extra_tokens = np.tile(per_row, [image_token_length_h])
+        joint = [
+                    [image_start_token_id],
+                    extra_tokens,
+                    [image_end_token_id],
+                ] + joint
+        joint = np.concatenate(joint, 0)
+        img_mask = np.pad(img_mask, [[0, 1], [0, 0]], constant_values=-1)
+        return patches, joint, patch_ordering, img_mask
+    def build_image_input_idx(
+        self,
+        image_tokens: np.ndarray,
+        patch_order: np.ndarray,
+        image_patch_token_id: int,
+        no_image: Optional[bool] = None,
+        image_token_length_w: Optional[int] = None,
+        image_token_length_h: Optional[int] = None,
+    ):
+        """Converts `patch_order` into a mapping of token_id -> patch_id"""
+        tokens_per_image = image_token_length_w * image_token_length_h
+        if no_image is not None and no_image:
+            return np.zeros((0, tokens_per_image), np.int32)
+        # Indices to insert the patches
+        image_input_idx = image_tokens == image_patch_token_id
+        image_input_idx = np.nonzero(image_input_idx)[0].astype(np.int32)
+        if patch_order is not None:
+            n_tokens = image_input_idx.shape[0]
+            patch_order = np.reshape(patch_order, [-1])
+            n_patches = patch_order.shape[0]
+            valid = patch_order >= 0
+            n_valid_patches = valid.sum()
+            assert len(image_input_idx) == n_valid_patches
+            sorted_patch_ixs = np.zeros([n_tokens], np.int32)
+            sorted_patch_ixs[patch_order[valid]] = np.arange(n_valid_patches, dtype=np.int32)
+            # Project the inverted mapping into same sparse structure
+            sorted_patch_ixs_ex = np.full(np.shape(patch_order), -1)
+            sorted_patch_ixs_ex[valid] = sorted_patch_ixs
+            # Do the gather and then re-masked outputs that were masked in `sorted_patch_ixs`
+            valid = (sorted_patch_ixs_ex >= 0).astype(np.int32)
+            image_input_idx = image_input_idx[sorted_patch_ixs_ex*valid]
+            image_input_idx = image_input_idx*valid - 100*(1 - valid)
+            image_input_idx = np.reshape(image_input_idx, [-1, tokens_per_image])
+        return image_input_idx
+    def preprocess(
+        self,
+        image: np.ndarray,
+        image_patch_token_id: int,
+        image_col_token_id: int,
+        image_start_token_id: int,
+        image_end_token_id: int,
+        max_crops: Optional[int] = None,
+        overlap_margins: Optional[List[int]] = None,
+        base_image_input_size: Optional[Union[int, List[int]]] = None,
+        image_token_length_w: Optional[int] = None,
+        image_token_length_h: Optional[int] = None,
+        image_patch_size: Optional[int] = None,
+        **kwargs,
+    ):
+        """Preprocesses an image
+        Returns:
+            crops: (n_crops, n_patches, patch_dim) individual crops, `n_crops` might
+                   change between images but the other dimension are fixed
+            tokens: (n_tokens,) int32 tokens, pad tokens indicate where to insert the
+                                patch features, might include other special tokens as well
+            image_idx: (n_crops, n_patches) index in `tokens` to put the patch features from the
+                       crops after pooling, negative values indicates patches features to exclude
+            padding_mask: (n_crops, n_patches) what percent of each crop is padding, can be None
+                          if the image mask is not being used.
+        """
+        max_crops = max_crops or self.max_crops
+        overlap_margins = overlap_margins or self.overlap_margins
+        base_image_input_size = base_image_input_size or self.base_image_input_size
+        image_token_length_w = image_token_length_w or self.image_token_length_w
+        image_token_length_h = image_token_length_h or self.image_token_length_h
+        image_patch_size = image_patch_size or self.image_patch_size
+        crops, image_tokens, patch_ordering, img_mask = self.image_to_patches_and_tokens(
+            image,
+            image_patch_token_id,
+            image_col_token_id,
+            image_start_token_id,
+            image_end_token_id,
+            max_crops,
+            overlap_margins,
+            base_image_input_size,
+            image_token_length_w,
+            image_token_length_h,
+            image_patch_size,
+        )
+        patch_idx = self.build_image_input_idx(
+            image_tokens,
+            patch_ordering,
+            image_patch_token_id,
+            image_token_length_w=image_token_length_w,
+            image_token_length_h=image_token_length_h,
+        )
+        return crops, image_tokens, patch_idx, img_mask
+    def multimodal_preprocess(
+        self,
+        images: np.ndarray,
+        tokens: List[int],
+        image_idx: np.ndarray,
+        sequence_length: int,
+        image_patch_token_id: int,
+        image_col_token_id: int,
+        image_start_token_id: int,
+        image_end_token_id: int,
+        **kwargs,
+    ):
+        """Merge images and text tokens into multi-modal features for the model
+        :param images: images to use as input
+        :param tokens: input text tokens
+        :param image_idx: where to insert the images into `tokens`
+        :params image_patch_token_id: id to use of tokens that will contain image features
+        :params image_col_token_id: token id for image column special tokens
+        :params image_start_token_id: token id for image start special tokens
+        :params image_end_token_id: token id for image end special tokens
+        :params kwargs: override preprocessor default args
+        """
+        max_total_crops = kwargs.get("max_crops") or self.max_crops
+        image_token_length_w = kwargs.get("image_token_length_w") or self.image_token_length_w
+        image_token_length_h = kwargs.get("image_token_length_h") or self.image_token_length_h
+        image_patch_size = kwargs.get("image_patch_size") or self.image_patch_size
+        base_image_input_size = kwargs.get("base_image_input_size") or self.base_image_input_size
+        image_num_patch = (
+            base_image_input_size[0] // image_patch_size,
+            base_image_input_size[1] // image_patch_size,
+        )
+        image_padding_mask = kwargs.get("image_padding_mask") or self.image_padding_mask
+        tokens_per_image = image_token_length_w * image_token_length_h
+        n_pixels = image_patch_size * image_patch_size * 3
+        n_patches = image_num_patch[0] * image_num_patch[1]
+        if images is None:
+            return {
+                "input_ids": tokens,
+            }
+        else:
+            n = len(images)
+            all_crops = []
+            all_image_idx = []
+            out_tokens = []
+            all_crop_masks = []
+            for ix in range(n):
+                token_ix = image_idx[ix]
+                crops, image_tokens, patch_idx, img_mask = self.preprocess(
+                    images[ix],
+                    image_patch_token_id,
+                    image_col_token_id,
+                    image_start_token_id,
+                    image_end_token_id,
+                    **kwargs,
+                )
+                if token_ix == -1:  # -1 is an image inserted at the very start
+                    start = 0
+                    token_ix = 0
+                    end = 0
+                else:
+                    start = 0 if ix == 0 else image_idx[ix-1] + 1
+                    end = token_ix + 1
+                all_image_idx.append(patch_idx + token_ix)
+                all_crops.append(crops)
+                out_tokens.append(tokens[start:token_ix])
+                out_tokens.append(image_tokens)
+                if ix == (n - 1):
+                    out_tokens.append(tokens[end:])
+                if image_padding_mask:
+                    all_crop_masks.append(img_mask)
+            input_ids = np.concatenate(out_tokens, 0)
+            images = np.concatenate(all_crops, 0)
+            image_input_idx = np.concatenate(all_image_idx, 0)
+            if image_padding_mask:
+                image_masks = np.concatenate(all_crop_masks, 0)
+            else:
+                image_masks = None
+        out = {
+            "input_ids": input_ids,
+            "images": images,
+            "image_input_idx": image_input_idx
+        }
+        if image_masks is not None:
+            out["image_masks"] = image_masks
+        return out
+MolmoImageProcessor.register_for_auto_class()

merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

model-00001-of-00005.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:6c4f011b584c85493adec12eee3cbd9f7f3df40ea7252a3dba801a394552ed17
+size 3961128688

model-00002-of-00005.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:1d7aeb98f5c48bb2c703d211195f81d75dbd792dddabd87ee6e7f5c88fd122ea
+size 3864722032

model-00003-of-00005.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7ca8b7dbc02260b365054c5e9ebffdd4e032366ac3d0798bd101aa75f4b285ad
+size 3728931368

model-00004-of-00005.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:88ffcf43f09e6a0c7f2e01254a65637c346bc6a2c161c9273ac2cc57dd51789b
+size 3996192344

model-00005-of-00005.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:daf0f1d9eb2a5fffbe316aa8b8baa08d88166a6c048eca125c272d28408dfacb
+size 520688944

model.safetensors.index.json ADDED Viewed

	@@ -0,0 +1,592 @@

+{
+  "metadata": {
+    "total_size": 16071582720
+  },
+  "weight_map": {
+    "model.transformer.blocks.0.att_proj.bias": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.0.att_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.0.attn_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.0.attn_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.0.ff_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.0.ff_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.0.ff_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.1.att_proj.bias": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.1.att_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.1.attn_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.1.attn_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.1.ff_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.1.ff_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.1.ff_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.10.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.10.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.10.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.10.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.10.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.10.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.10.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.11.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.11.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.11.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.11.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.11.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.11.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.11.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.12.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.12.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.12.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.12.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.12.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.12.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.12.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.13.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.13.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.13.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.13.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.13.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.13.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.13.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.14.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.14.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.14.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.14.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.14.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.14.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.14.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.15.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.15.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.15.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.15.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.15.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.15.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.15.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.16.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.16.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.16.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.16.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.16.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.16.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.16.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.17.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.17.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.17.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.17.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.17.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.17.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.17.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.18.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.18.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.18.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.18.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.18.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.18.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.18.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.19.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.19.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.19.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.19.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.19.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.19.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.19.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.2.att_proj.bias": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.2.att_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.2.attn_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.2.attn_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.2.ff_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.2.ff_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.2.ff_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.20.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.20.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.20.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.20.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.20.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.20.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.20.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.21.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.21.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.21.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.21.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.21.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.21.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.21.ff_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.22.att_proj.bias": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.22.att_proj.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.22.attn_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.22.attn_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.22.ff_norm.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.22.ff_out.weight": "model-00003-of-00005.safetensors",
+    "model.transformer.blocks.22.ff_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.23.att_proj.bias": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.23.att_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.23.attn_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.23.attn_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.23.ff_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.23.ff_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.23.ff_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.24.att_proj.bias": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.24.att_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.24.attn_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.24.attn_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.24.ff_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.24.ff_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.24.ff_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.25.att_proj.bias": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.25.att_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.25.attn_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.25.attn_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.25.ff_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.25.ff_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.25.ff_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.26.att_proj.bias": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.26.att_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.26.attn_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.26.attn_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.26.ff_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.26.ff_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.26.ff_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.27.att_proj.bias": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.27.att_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.27.attn_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.27.attn_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.27.ff_norm.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.27.ff_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.27.ff_proj.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.blocks.3.att_proj.bias": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.3.att_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.3.attn_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.3.attn_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.3.ff_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.3.ff_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.3.ff_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.4.att_proj.bias": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.4.att_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.4.attn_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.4.attn_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.4.ff_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.4.ff_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.4.ff_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.5.att_proj.bias": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.5.att_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.5.attn_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.5.attn_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.5.ff_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.5.ff_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.5.ff_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.6.att_proj.bias": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.6.att_proj.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.6.attn_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.6.attn_out.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.6.ff_norm.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.blocks.6.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.6.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.7.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.7.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.7.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.7.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.7.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.7.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.7.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.8.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.8.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.8.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.8.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.8.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.8.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.8.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.9.att_proj.bias": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.9.att_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.9.attn_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.9.attn_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.9.ff_norm.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.9.ff_out.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.blocks.9.ff_proj.weight": "model-00002-of-00005.safetensors",
+    "model.transformer.ff_out.weight": "model-00004-of-00005.safetensors",
+    "model.transformer.ln_f.weight": "model-00001-of-00005.safetensors",
+    "model.transformer.wte.embedding": "model-00001-of-00005.safetensors",
+    "model.transformer.wte.new_embedding": "model-00001-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wk.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wk.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wo.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wo.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wq.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wq.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wv.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_pooling_2d.wv.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_projector.w1.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_projector.w2.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_projector.w3.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.class_embedding": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.patch_embedding.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.positional_embedding": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.pre_ln.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.pre_ln.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wk.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wk.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wo.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wo.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wq.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wq.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wv.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention.wv.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention_norm.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.attention_norm.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.feed_forward.w1.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.feed_forward.w1.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.feed_forward.w2.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.feed_forward.w2.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.ffn_norm.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.0.ffn_norm.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wk.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wk.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wo.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wo.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wq.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wq.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wv.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention.wv.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention_norm.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.attention_norm.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.feed_forward.w1.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.feed_forward.w1.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.feed_forward.w2.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.feed_forward.w2.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.ffn_norm.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.1.ffn_norm.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wk.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wk.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wo.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wo.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wq.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wq.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wv.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention.wv.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention_norm.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.attention_norm.weight": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.feed_forward.w1.bias": "model-00004-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.10.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.11.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.12.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.13.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.14.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.15.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.16.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.17.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.18.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.19.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.2.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.20.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.21.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.22.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.3.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.4.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.5.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.6.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.7.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.8.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wk.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wk.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wo.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wo.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wq.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wq.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wv.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention.wv.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.attention_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.feed_forward.w1.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.feed_forward.w1.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.feed_forward.w2.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.feed_forward.w2.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.ffn_norm.bias": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.image_vit.transformer.resblocks.9.ffn_norm.weight": "model-00005-of-00005.safetensors",
+    "model.vision_backbone.pad_embed": "model-00005-of-00005.safetensors"
+  }
+}

modeling_molmo.py ADDED Viewed

	@@ -0,0 +1,2371 @@

+import logging
+import math
+from copy import deepcopy
+from dataclasses import fields, dataclass, replace
+from enum import Enum
+from typing import List, Optional, Tuple, Union, Dict, Any, Sequence, Callable, cast, MutableMapping
+import torch
+from einops import einsum, einops
+from transformers import PreTrainedModel, GenerationConfig
+from transformers.cache_utils import Cache
+from transformers.modeling_outputs import CausalLMOutputWithPast, ModelOutput
+from transformers.models.auto import AutoModelForCausalLM
+from torch import nn
+from .config_molmo import MolmoConfig
+from torch.nn import functional as F
+log = logging.getLogger(__name__)
+class BufferCache(dict, MutableMapping[str, torch.Tensor]):
+    """
+    Cache for attention biases and other things that would normally be stored as buffers.
+    We avoid using buffers because we've run into various issues doing so with FSDP.
+    In general it appears the way FSDP handles buffers is not well-defined.
+    It doesn't shard them but apparently it does synchronize them across processes, which we want to avoid
+    since (A) it isn't necessary, and (B) we sometimes have `-inf` in these biases which might get turned into
+    NaNs when they're synchronized due to casting or some other issue.
+    """
+class StrEnum(str, Enum):
+    def __str__(self) -> str:
+        return self.value
+    def __repr__(self) -> str:
+        return f"'{str(self)}'"
+class ImageProjectType(StrEnum):
+    mlp = "mlp"
+    mlpx2 = "2mlp"
+    linear = "linear"
+class ImagePooling2DType(StrEnum):
+    attention = "attention"
+    attention_meanq = "attention-meanq"
+    attention_2wide = "attention_2wide"
+    attention_v2 = "attention-v2"
+    none = "none"
+    stack = "stack"
+class ActivationType(StrEnum):
+    quick_gelu = "quick_gelu"
+    gelu = "gelu"
+    gelu_tanh = "gelu_tanh"
+    relu = "relu"
+    silu = "silu"
+    llama_geglu = "llama_geglu"
+    llama_geglu_tanh = "llama_geglu_tanh"
+    llama_swiglu = "llama_swiglu"
+    swiglu = "swiglu"
+def ensure_finite_(x: torch.Tensor, check_neg_inf: bool = True, check_pos_inf: bool = False):
+    """
+    Modify ``x`` in place to replace ``float("-inf")`` with the minimum value of the dtype when ``check_neg_inf``
+    is ``True`` and to replace ``float("inf")`` with the maximum value of the dtype when ``check_pos_inf`` is ``True``.
+    """
+    if check_neg_inf:
+        x.masked_fill_(x == float("-inf"), torch.finfo(x.dtype).min)
+    if check_pos_inf:
+        x.masked_fill_(x == float("inf"), torch.finfo(x.dtype).max)
+class MolmoConfigurationError(Exception):
+    pass
+def _non_meta_init_device(config) -> torch.device:
+    if config.init_device is not None and config.init_device != "meta":
+        return torch.device(config.init_device)
+    else:
+        return torch.device("cuda" if torch.cuda.is_available() else "cpu")
+class RotaryEmbedding(nn.Module):
+    """
+    [Rotary positional embeddings (RoPE)](https://arxiv.org/abs/2104.09864).
+    """
+    def __init__(self, config: MolmoConfig, cache: BufferCache):
+        super().__init__()
+        self.config = config
+        self.__cache = cache
+        # Warm up cache.
+        self.get_rotary_embedding(
+            config.max_position_embeddings or config.max_sequence_length,
+            _non_meta_init_device(config)
+        )
+    def get_rotary_embedding(self, seq_len: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]:
+        if (
+            (pos_sin := self.__cache.get("rope_pos_sin")) is not None
+            and (pos_cos := self.__cache.get("rope_pos_cos")) is not None
+            and pos_sin.shape[-2] >= seq_len
+            and pos_cos.shape[-2] >= seq_len
+        ):
+            if pos_sin.device != device:
+                pos_sin = pos_sin.to(device)
+                self.__cache["rope_pos_sin"] = pos_sin
+            if pos_cos.device != device:
+                pos_cos = pos_cos.to(device)
+                self.__cache["rope_pos_cos"] = pos_cos
+            return pos_sin[:, :, :seq_len, :], pos_cos[:, :, :seq_len, :]
+        with torch.autocast(device.type, enabled=False):
+            dim = self.config.d_model // self.config.n_heads
+            inv_freq = 1.0 / (self.config.rope_theta ** (torch.arange(0, dim, 2, device=device, dtype=torch.float) / dim))
+            seq = torch.arange(seq_len, device=device, dtype=torch.float)
+            freqs = torch.einsum("i , j -> i j", seq, inv_freq)
+            if self.config.rope_impl == "interleave":
+                positions = freqs.repeat_interleave(2, dim=-1)
+            else:
+                positions = torch.cat((freqs, freqs), dim=-1)
+            pos_sin, pos_cos = positions.sin()[None, None, :, :], positions.cos()[None, None, :, :]
+        self.__cache["rope_pos_sin"] = pos_sin
+        self.__cache["rope_pos_cos"] = pos_cos
+        return pos_sin, pos_cos
+    def rotate_half(self, x: torch.Tensor) -> torch.Tensor:
+        B, nh, T, hs = x.size()
+        x = x.view(B, nh, T, 2, hs // 2)
+        x1, x2 = x.unbind(dim=-2)
+        return torch.cat((-x2, x1), dim=-1)
+    def rotate_every_two(self, x: torch.Tensor) -> torch.Tensor:
+        B, nh, T, hs = x.size()
+        x = x.view(B, nh, T, hs // 2, 2)
+        x1, x2 = x.unbind(dim=-1)
+        x = torch.stack((-x2, x1), dim=-1)
+        return x.view(B, nh, T, hs)
+    def apply_rotary_pos_emb(self, pos_sin: torch.Tensor, pos_cos: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
+        if self.config.rope_impl == "interleave":
+            return ((t * pos_cos) + (self.rotate_every_two(t) * pos_sin)).to(t.dtype)
+        else:
+            return ((t * pos_cos) + (self.rotate_half(t) * pos_sin)).to(t.dtype)
+    def forward(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        position_ids: Optional[torch.Tensor] = None
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        if self.config.rope_full_precision:
+            q_, k_ = q.float(), k.float()
+        else:
+            q_, k_ = q, k
+        with torch.autocast(q.device.type, enabled=False):
+            batch_size = q_.shape[0]
+            query_len, key_len = q_.shape[-2], k_.shape[-2]  # could be different if layer_past not None
+            if position_ids is not None:
+                freqs_cis_len = (self.config.max_position_embeddings or self.config.max_sequence_length)
+            else:
+                freqs_cis_len = key_len
+            pos_sin, pos_cos = self.get_rotary_embedding(freqs_cis_len, q_.device)
+            pos_sin = pos_sin.type_as(q_)
+            pos_cos = pos_cos.type_as(q_)
+            if position_ids is not None:
+                assert query_len == key_len, "Query and key lengths must be equal when using position IDs."
+                pos_sin = pos_sin[0, 0][position_ids].view(
+                    (batch_size, 1, key_len, pos_sin.shape[-1])
+                )
+                pos_cos = pos_cos[0, 0][position_ids].view(
+                    (batch_size, 1, key_len, pos_cos.shape[-1])
+                )
+            q_ = self.apply_rotary_pos_emb(
+                pos_sin[:, :, key_len - query_len : key_len, :],
+                pos_cos[:, :, key_len - query_len : key_len, :],
+                q_,
+            )
+            k_ = self.apply_rotary_pos_emb(pos_sin, pos_cos, k_)
+        return q_.type_as(q), k_.type_as(k)
+class MolmoBlock(nn.Module):
+    """
+    A base class for transformer block implementations.
+    """
+    def __init__(self, layer_id: int, config: MolmoConfig, cache: BufferCache):
+        super().__init__()
+        self.layer_id = layer_id
+        self.config = config
+        self.hidden_size = (
+            config.mlp_hidden_size if config.mlp_hidden_size is not None else config.mlp_ratio * config.d_model
+        )
+        self.__cache = cache
+        self._activation_checkpoint_fn = None
+        # Dropout.
+        self.dropout = Dropout(config.residual_dropout)
+        # Layer norms.
+        self.k_norm: Optional[LayerNormBase] = None
+        self.q_norm: Optional[LayerNormBase] = None
+        if config.attention_layer_norm:
+            assert config.effective_n_kv_heads is not None
+            self.k_norm = LayerNormBase.build(
+                config,
+                size=(config.d_model // config.n_heads) * config.effective_n_kv_heads,
+                elementwise_affine=config.attention_layer_norm_with_affine,
+            )
+            self.q_norm = LayerNormBase.build(config, elementwise_affine=config.attention_layer_norm_with_affine)
+        # Make sure QKV clip coefficient is positive, otherwise it's not well-defined.
+        if config.clip_qkv is not None:
+            assert config.clip_qkv > 0
+        # Activation function.
+        self.act = Activation.build(config)
+        assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
+        # Attention output projection.
+        input_dim = config.d_model
+        self.attn_out = nn.Linear(
+            input_dim, config.d_model,
+            bias=config.include_bias,
+            device=config.init_device
+        )
+        # Feed-forward output projection.
+        self.ff_out = nn.Linear(
+            int(self.act.output_multiplier * self.hidden_size),
+            config.d_model,
+            bias=config.include_bias,
+            device=config.init_device,
+        )
+        self.ff_out._is_residual = True  # type: ignore
+        # Rotary embeddings.
+        if self.config.rope:
+            self.rotary_emb = RotaryEmbedding(config, self.__cache)
+        self.flash_attn_func = None
+        if config.attention_type == "flash":
+            try:
+                from flash_attn import flash_attn_func  # type: ignore
+                self.flash_attn_func = flash_attn_func
+            except ModuleNotFoundError:
+                pass
+    def reset_parameters(self):
+        if self.k_norm is not None:
+            self.k_norm.reset_parameters()
+        if self.q_norm is not None:
+            self.q_norm.reset_parameters()
+        init_weights(
+            self.config,
+            self.attn_out,
+            d=self.config.d_model,
+            layer_id=self.layer_id,
+            type_of_module=ModuleType.out_module,
+        )
+        init_weights(
+            self.config,
+            self.ff_out,
+            d=self.ff_out.in_features,
+            layer_id=self.layer_id,
+            type_of_module=ModuleType.out_module,
+        )
+    @classmethod
+    def _cast_attn_bias(cls, bias: torch.Tensor, input_dtype: torch.dtype) -> torch.Tensor:
+        target_dtype = input_dtype
+        # NOTE: `is_autocast_enabled()` only checks for CUDA autocast, so we use the separate function
+        # `is_autocast_cpu_enabled()` for CPU autocast.
+        # See https://github.com/pytorch/pytorch/issues/110966.
+        if bias.device.type == "cuda" and torch.is_autocast_enabled():
+            target_dtype = torch.get_autocast_gpu_dtype()
+        elif bias.device.type == "cpu" and torch.is_autocast_cpu_enabled():
+            target_dtype = torch.get_autocast_cpu_dtype()
+        if bias.dtype != target_dtype:
+            bias = bias.to(target_dtype)
+            ensure_finite_(bias, check_neg_inf=True, check_pos_inf=False)
+        return bias
+    def _scaled_dot_product_attention(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        attn_mask: Optional[torch.Tensor] = None,
+        dropout_p: float = 0.0,
+        response_dropout_p: float = 0.0,
+        is_causal: bool = False,
+    ) -> torch.Tensor:
+        """
+        Computes scaled dot product attention on query, key and value tensors, using an optional
+        attention mask if passed, and applying dropout if a probability greater than 0.0 is specified.
+        """
+        if attn_mask is not None:
+            attn_mask = attn_mask.to(q.device)
+        if self.flash_attn_func is not None and attn_mask is None:
+            r = self.flash_attn_func(
+                q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), dropout_p=dropout_p, causal=is_causal
+            )
+            return r.transpose(1, 2)
+        else:
+            # torch's sdpa doesn't support GQA, so we're doing this
+            assert k.size(1) == v.size(1)
+            num_kv_heads = k.size(1)
+            num_q_heads = q.size(1)
+            if num_q_heads != num_kv_heads:
+                assert num_q_heads % num_kv_heads == 0
+                k = k.repeat_interleave(num_q_heads // num_kv_heads, dim=1, output_size=num_q_heads)
+                v = v.repeat_interleave(num_q_heads // num_kv_heads, dim=1, output_size=num_q_heads)
+            return F.scaled_dot_product_attention(
+                q,
+                k,
+                v,
+                attn_mask=attn_mask,
+                dropout_p=dropout_p,
+                is_causal=is_causal,
+            )
+    def attention(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        attention_bias: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        B, T, C = q.size()  # batch size, sequence length, d_model
+        dtype = k.dtype
+        # Optionally apply layer norm to keys and queries.
+        if self.q_norm is not None and self.k_norm is not None:
+            q = self.q_norm(q).to(dtype=dtype)
+            k = self.k_norm(k).to(dtype=dtype)
+        # Move head forward to be next to the batch dim.
+        # shape: (B, nh, T, hs)
+        q = q.view(B, T, self.config.n_heads, C // self.config.n_heads).transpose(1, 2)
+        # shape: (B, n_kv_h, T, hs)
+        k = k.view(B, T, self.config.effective_n_kv_heads, C // self.config.n_heads).transpose(1, 2)
+        # shape: (B, n_kv_h, T, hs)
+        v = v.view(B, T, self.config.effective_n_kv_heads, C // self.config.n_heads).transpose(1, 2)
+        if self.config.use_position_ids and self.config.rope:
+            # Apply rotary embeddings
+            q, k = self.rotary_emb(q, k, position_ids=position_ids)
+        if layer_past is not None:
+            past_key, past_value = layer_past
+            k = torch.cat((past_key.to(k.device), k), dim=-2)
+            v = torch.cat((past_value.to(v.device), v), dim=-2)
+        present = (k, v) if use_cache else None
+        query_len, key_len = q.shape[-2], k.shape[-2]  # could be different if layer_past not None
+        if not self.config.use_position_ids and self.config.rope:
+            # Apply rotary embeddings
+            q, k = self.rotary_emb(q, k)
+        if attention_bias is not None:
+            # Resize and cast attention bias.
+            # The current dtype of the attention bias might not match the dtype that the SDP attn function will
+            # run in if AMP is enabled, and this can be a problem if some tokens are masked out due to padding
+            # as down-casting the attention bias to the autocast precision will result in -infs, which will
+            # cause the SDP attn function to produce NaNs.
+            attention_bias = self._cast_attn_bias(
+                attention_bias[:, :, key_len - query_len : key_len, :key_len], dtype
+            )
+        # Get the attention scores.
+        # shape: (B, nh, T, hs)
+        att = self._scaled_dot_product_attention(
+            q,
+            k,
+            v,
+            attn_mask=attention_bias,
+            dropout_p=0.0 if not self.training else self.config.attention_dropout,
+            response_dropout_p=0.0 if not self.training else self.config.response_attention_dropout,
+            is_causal=attention_bias is None,
+        )
+        # Re-assemble all head outputs side-by-side.
+        att = att.transpose(1, 2).contiguous().view(B, T, C)
+        # Apply output projection.
+        return self.attn_out(att), present
+    def forward(
+        self,
+        x: torch.Tensor,
+        attention_bias: Optional[torch.FloatTensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        raise NotImplementedError
+    @classmethod
+    def build(cls, layer_id: int, config: MolmoConfig, cache: BufferCache):
+        return MolmoSequentialBlock(layer_id, config, cache)
+class MolmoSequentialBlock(MolmoBlock):
+    """
+    This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
+    (plus another skip connection).
+    """
+    def __init__(self, layer_id: int, config: MolmoConfig, cache: BufferCache):
+        super().__init__(layer_id, config, cache)
+        # Layer norms.
+        self.attn_norm = LayerNorm.build(config)
+        self.ff_norm = LayerNorm.build(config)
+        # Attention input projection. Projects x -> (q, k, v)
+        head_dim = config.d_model // config.n_heads
+        self.fused_dims = (
+            config.d_model,
+            config.effective_n_kv_heads * head_dim,
+            config.effective_n_kv_heads * head_dim,
+        )
+        self.att_proj = nn.Linear(
+            config.d_model, sum(self.fused_dims),
+            bias=config.include_bias or config.qkv_bias,
+            device=config.init_device
+        )
+        # Feed-forward input projection.
+        self.ff_proj = nn.Linear(
+            config.d_model, self.hidden_size, bias=config.include_bias, device=config.init_device
+        )
+    def reset_parameters(self):
+        super().reset_parameters()
+        self.attn_norm.reset_parameters()
+        self.ff_norm.reset_parameters()
+        # NOTE: the standard deviation for these weights does not depend on the layer.
+        init_weights(
+            self.config, self.att_proj, d=self.config.d_model, layer_id=None, type_of_module=ModuleType.in_module
+        )
+        init_weights(
+            self.config, self.ff_proj, d=self.config.d_model, layer_id=None, type_of_module=ModuleType.in_module
+        )
+    def forward(
+        self,
+        x: torch.Tensor,
+        attention_bias: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        # Get query, key, value projections.
+        # shape:
+        #  - for regular attn q, k, v: (batch_size, seq_len, d_model)
+        #  - for multi-query attn q: (batch_size, seq_len, d_model)
+        #                      k, v: (batch_size, seq_len, d_model // n_heads)
+        #  - for group query attn q: (batch_size, seq_len, d_model)
+        #                      k, v: (batch_size, seq_len, d_model // n_kv_heads)
+        if not self.config.norm_after:
+            if self._activation_checkpoint_fn is not None:
+                atten_in = self._activation_checkpoint_fn(self.attn_norm, x)
+            else:
+                atten_in = self.attn_norm(x)
+        else:
+            atten_in = x
+        qkv = self.att_proj(atten_in)
+        if self.config.clip_qkv is not None:
+            qkv.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+        q, k, v = qkv.split(self.fused_dims, dim=-1)
+        # Get attention scores.
+        if self._activation_checkpoint_fn is not None:
+            att, cache = self._activation_checkpoint_fn(  # type: ignore
+                self.attention, q, k, v, attention_bias, position_ids=position_ids, layer_past=layer_past, use_cache=use_cache
+            )
+        else:
+            att, cache = self.attention(q, k, v, attention_bias, position_ids=position_ids, layer_past=layer_past, use_cache=use_cache)
+        if self.config.norm_after:
+            if self._activation_checkpoint_fn is not None:
+                att = self._activation_checkpoint_fn(self.attn_norm, att)
+            else:
+                att = self.attn_norm(att)
+        # Add attention scores.
+        # shape: (B, T, C)
+        x = x + self.dropout(att)
+        # Add feed-forward projection.
+        # shape: (batch_size, seq_len, d_model)
+        og_x = x
+        if not self.config.norm_after:
+            if self._activation_checkpoint_fn is not None:
+                x = self._activation_checkpoint_fn(self.ff_norm, x)  # type: ignore
+            else:
+                x = self.ff_norm(x)
+        x = self.ff_proj(x)
+        if self._activation_checkpoint_fn is not None:
+            x = self._activation_checkpoint_fn(self.act, x)  # type: ignore
+        else:
+            x = self.act(x)
+        x = self.ff_out(x)
+        if self.config.norm_after:
+            if self._activation_checkpoint_fn is not None:
+                x = self._activation_checkpoint_fn(self.ff_norm, x)  # type: ignore
+            else:
+                x = self.ff_norm(x)
+        x = self.dropout(x)
+        x = og_x + x
+        return x, cache
+class Embedding(nn.Module):
+    def __init__(
+        self,
+        num_embeddings: int,
+        num_new_embeddings: int,
+        features: int,
+        device: Union[str, torch.device],
+        initializer_range: float = 0.02,
+        new_embed_initializer_range: float = 0.02,
+    ):
+        super().__init__()
+        self.initializer_range = initializer_range
+        self.new_embed_initializer_range = new_embed_initializer_range
+        self.embedding = nn.Parameter(
+            torch.zeros(num_embeddings, features, device=device),
+        )
+        self.new_embedding = nn.Parameter(
+            torch.zeros(num_new_embeddings, features, device=device),
+        )
+    def reset_parameters(self):
+        nn.init.normal_(self.embedding, std=self.initializer_range)
+        nn.init.normal_(self.new_embedding, std=self.new_embed_initializer_range)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return F.embedding(x, torch.cat([self.embedding, self.new_embedding], dim=0))
+class Dropout(nn.Dropout):
+    def __init__(
+        self,
+        p: float = 0.5,
+        inplace: bool = False,
+        mask_p: float = 0,
+        broadcast_dims: Sequence[int] = (),
+    ):
+        super().__init__(p, inplace)
+        self.mask_p = mask_p
+        self.broadcast_dims = broadcast_dims
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        """
+        :param input: A tensor of shape `(batch_size, seq_len, embed_dim)`
+        """
+        if self.p == 0.0 and (self.mask_p is None or self.mask_p == 0.0):
+            return input
+        else:
+            if self.p > 0. and len(self.broadcast_dims) > 0 and self.training:
+                keep_prob = 1.0 - self.p
+                dropout_shape = list(input.shape)
+                for dim in self.broadcast_dims:
+                    dropout_shape[dim] = 1
+                keep = input.new_empty(dropout_shape).bernoulli_(keep_prob)
+                multiplier = keep.broadcast_to(input.shape)
+                multiplier.div_(keep_prob)
+                input = input * multiplier
+            else:
+                return F.dropout(input, self.p, self.training, self.inplace)
+@dataclass
+class VisionBackboneConfig:
+    image_default_input_size: Tuple[int, int] = (336, 336)
+    image_patch_size: int = 14
+    image_pos_patch_size: int = 14
+    image_emb_dim: int = 1024
+    image_num_heads: int = 16
+    image_num_key_value_heads: int = 16
+    image_num_layers: int = 24
+    image_head_dim: int = 64
+    image_mlp_dim: int = 4096
+    image_mlp_activations: str = "gelu"
+    image_dropout_rate: float = 0.0
+    image_num_pos: int = 577
+    image_norm_eps: float = 1e-5
+    attention_dropout: float = 0.0
+    residual_dropout: float = 0.0
+    initializer_range: float = 0.02
+    fsdp_wrap: bool = False
+    resize_mode: str = "default"
+    def __post_init__(self):
+        self.image_default_input_size = tuple(self.image_default_input_size)  # type: ignore[assignment]
+    @property
+    def image_num_patch(self):
+        h, w = self.image_default_input_size
+        return h // self.image_patch_size, w // self.image_patch_size
+@dataclass
+class FullMolmoConfig:
+    d_model: int = 768
+    n_heads: int = 12
+    n_kv_heads: Optional[int] = None
+    qkv_bias: bool = False
+    clip_qkv: Optional[float] = None
+    n_layers: int = 12
+    mlp_ratio: int = 4
+    mlp_hidden_size: Optional[int] = None
+    activation_type: str = "swiglu"
+    block_group_size: int = 1
+    rope: bool = True
+    rope_full_precision: bool = True
+    rope_theta: float = 10000.
+    rope_impl: str = "interleave"
+    vision_backbone: Optional[VisionBackboneConfig] = None
+    attention_type: str = "sdpa"
+    float32_attention: bool = True
+    attention_dropout: float = 0.1
+    response_attention_dropout: float = 0.0
+    multi_query_attention: Optional[bool] = None
+    attention_layer_norm: bool = False
+    residual_dropout: float = 0.1
+    embedding_dropout: float = 0.1
+    layer_norm_type: str = "default"
+    layer_norm_with_affine: bool = True
+    layer_norm_eps: Optional[float] = None
+    attention_layer_norm_with_affine: bool = True
+    max_sequence_length: int = 1024
+    max_position_embeddings: Optional[int] = None
+    include_bias: bool = True
+    bias_for_layer_norm: Optional[bool] = None
+    scale_logits: bool = False
+    vocab_size: int = 50257
+    embedding_size: Optional[int] = 50304
+    additional_vocab_size: Optional[int] = None
+    new_embedding_init_range: float = 0.02
+    weight_tying: bool = True
+    pad_token_id: int = -1
+    init_device: Optional[str] = None
+    init_std: float = 0.02
+    init_cutoff_factor: Optional[float] = None
+    norm_after: bool = False
+    precision: Optional[str] = None
+    image_padding_embed: Optional[str] = None
+    vit_layers: Tuple = (-1,)
+    image_pooling_h: int = 2
+    image_pooling_w: int = 2
+    image_pooling_2d: str = "attention"
+    image_projector: str = "mlp"
+    image_feature_dropout: float = 0.0
+    initializer_range: float = 0.02
+    normalize_input_embeds: bool = False
+    use_position_ids: bool = True
+    @property
+    def effective_n_kv_heads(self) -> int:
+        if self.n_kv_heads is None:
+            if self.multi_query_attention is True:
+                return 1
+            else:
+                return self.n_heads
+        else:
+            if self.multi_query_attention is None:
+                return self.n_kv_heads
+            if self.multi_query_attention:
+                n_kv_heads_should_be = 1
+            else:
+                n_kv_heads_should_be = self.n_heads
+            if self.n_kv_heads == n_kv_heads_should_be:
+                return n_kv_heads_should_be
+            else:
+                raise MolmoConfigurationError(
+                    "You can't set `multi_query_attention` and `n_kv_heads` at the same time."
+                )
+    @property
+    def image_num_patch(self):
+        assert self.vision_backbone is not None
+        return self.vision_backbone.image_num_patch
+    @property
+    def image_patch_size(self):
+        assert self.vision_backbone is not None
+        return self.visoin_backbone.image_patch_size
+    def llm_patches_per_crop(self):
+        h, w = self.image_num_patch
+        # Round up in case we need to pad the image features for pooling
+        h = (h + self.image_pooling_h - 1) // self.image_pooling_h
+        w = (w + self.image_pooling_w - 1) // self.image_pooling_w
+        return h, w
+def _expand_token(token, batch_size: int):
+    return token.view(1, 1, -1).expand(batch_size, -1, -1)
+class ViTMLP(nn.Module):
+    def __init__(self, config: FullMolmoConfig):
+        super().__init__()
+        self.config = config
+        v_cfg = config.vision_backbone
+        self.w1 = nn.Linear(
+            v_cfg.image_emb_dim,
+            v_cfg.image_mlp_dim,
+            bias=True,
+            device=config.init_device,
+        )
+        # Activation function.
+        cfg = deepcopy(config)
+        cfg.activation_type = v_cfg.image_mlp_activations
+        self.act = Activation.build(cfg)
+        self.w2 = nn.Linear(
+            v_cfg.image_mlp_dim,
+            v_cfg.image_emb_dim,
+            bias=True,
+            device=config.init_device,
+        )
+    def reset_parameters(self):
+        v_cfg = self.config.vision_backbone
+        nn.init.trunc_normal_(self.w1.weight, std=math.sqrt(1 / v_cfg.image_emb_dim), a=-2.0, b=2.0)
+        nn.init.trunc_normal_(self.w2.weight, std=math.sqrt(1 / v_cfg.image_mlp_dim), a=-2.0, b=2.0)
+        nn.init.zeros_(self.w1.bias)
+        nn.init.zeros_(self.w2.bias)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.w1(x)
+        x = self.act(x)
+        x = self.w2(x)
+        return x
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, config: FullMolmoConfig):
+        super().__init__()
+        self.config = config
+        v_cfg = config.vision_backbone
+        self.attention = MultiHeadDotProductAttention(config)
+        self.feed_forward = ViTMLP(config)
+        self.attention_norm = nn.LayerNorm(
+            v_cfg.image_emb_dim,
+            eps=v_cfg.image_norm_eps,
+            device=config.init_device,
+        )
+        self.ffn_norm = nn.LayerNorm(
+            v_cfg.image_emb_dim,
+            eps=v_cfg.image_norm_eps,
+            device=config.init_device,
+        )
+    def reset_parameters(self):
+        self.attention.reset_parameters()
+        self.feed_forward.reset_parameters()
+        self.attention_norm.reset_parameters()
+        self.ffn_norm.reset_parameters()
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = x + self.attention(self.attention_norm(x))
+        x = x + self.feed_forward(self.ffn_norm(x))
+        return x
+class BlockCollection(nn.Module):
+    def __init__(self, config: FullMolmoConfig):
+        super().__init__()
+        self.config = config
+        self.grad_checkpointing: bool = False
+        v_cfg = config.vision_backbone
+        self.resblocks = nn.ModuleList([
+            ResidualAttentionBlock(config) for _ in range(v_cfg.image_num_layers)
+        ])
+    def reset_parameters(self):
+        for r in self.resblocks:
+            r.reset_parameters()
+    def forward(self, x: torch.Tensor) -> List[torch.Tensor]:
+        hidden_states = []
+        for r in self.resblocks:
+            x = r(x)
+            hidden_states.append(x)
+        return hidden_states
+class LayerNormFp32(nn.LayerNorm):
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        orig_type = x.dtype
+        x = F.layer_norm(x.to(torch.float32), self.normalized_shape, self.weight.to(torch.float32),
+                         self.bias.to(torch.float32), self.eps)
+        return x.to(orig_type)
+class VisionTransformer(nn.Module):
+    def __init__(self, config: FullMolmoConfig):
+        super().__init__()
+        self.config = config
+        v_cfg = config.vision_backbone
+        # class embeddings and positional embeddings
+        self.scale = v_cfg.image_emb_dim ** -0.5
+        self.class_embedding = nn.Parameter(
+            torch.zeros(v_cfg.image_emb_dim, device=config.init_device),
+        )
+        self.num_prefix_tokens: int = 1
+        self.positional_embedding = nn.Parameter(
+            torch.zeros(v_cfg.image_num_pos, v_cfg.image_emb_dim, device=config.init_device),
+        )
+        image_patch_size = v_cfg.image_patch_size
+        self.patch_embedding = nn.Linear(
+            image_patch_size * image_patch_size * 3,
+            v_cfg.image_emb_dim,
+            bias=False,
+            device=config.init_device,
+            )
+        self.pre_ln = LayerNormFp32(
+            v_cfg.image_emb_dim,
+            eps=v_cfg.image_norm_eps,
+        )
+        self.transformer = BlockCollection(config)
+    @torch.jit.ignore
+    def set_grad_checkpointing(self, enable=True):
+        self.transformer.grad_checkpointing = enable
+    def reset_parameters(self):
+        nn.init.normal_(self.class_embedding, std=self.scale)
+        nn.init.normal_(self.positional_embedding, std=self.scale)
+        nn.init.normal_(self.patch_embedding.weight, std=0.02)
+        self.pre_ln.reset_parameters()
+        self.transformer.reset_parameters()
+    def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor:
+        cls_emb = self.positional_embedding[0:1]
+        pos_emb = self.positional_embedding[1:]
+        pos_emb = pos_emb.reshape(
+            (int(math.sqrt(pos_emb.shape[0])), int(math.sqrt(pos_emb.shape[0])), pos_emb.shape[1])
+        )
+        (patch_num_0, patch_num_1) = patch_num
+        if pos_emb.shape[0] != patch_num_0 or pos_emb.shape[1] != patch_num_1:
+            # Dervied from https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
+            # antialias: default True in jax.image.resize
+            pos_emb = pos_emb.unsqueeze(0).permute(0, 3, 1, 2)
+            pos_emb = F.interpolate(
+                pos_emb, size=(patch_num_0, patch_num_1), mode="bicubic", align_corners=False, antialias=True,
+            )
+            pos_emb = pos_emb.permute(0, 2, 3, 1).squeeze(0)
+        pos_emb = pos_emb.reshape(-1, pos_emb.shape[-1])
+        x = x + torch.cat([cls_emb[None, :, :], pos_emb[None, :, :]], dim=1).to(x.dtype)
+        return x
+    def forward(self, x: torch.Tensor, patch_num: int = None) -> List[torch.Tensor]:
+        """
+        : param x: (batch_size, num_patch, n_pixels)
+        """
+        if patch_num is None:
+            patch_num = self.config.vision_backbone.image_num_patch
+        B, N, D = x.shape
+        x = self.patch_embedding(x)
+        # class embeddings and positional embeddings
+        x = torch.cat([_expand_token(self.class_embedding, x.shape[0]).to(x.dtype), x], dim=1)
+        x = self.add_pos_emb(x, patch_num)
+        x = self.pre_ln(x)
+        hidden_states = self.transformer(x)
+        return hidden_states
+class MultiHeadDotProductAttention(nn.Module):
+    def __init__(self, config: FullMolmoConfig, use_bias: bool = True, is_vit_layer: Optional[bool] = True):
+        super().__init__()
+        self.config = config
+        self.use_bias = use_bias
+        v_cfg = config.vision_backbone
+        self.embed_dim = v_cfg.image_emb_dim
+        self.num_heads = v_cfg.image_num_heads
+        self.head_dim = v_cfg.image_head_dim
+        self.num_key_value_heads = v_cfg.image_num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.initializer_range = v_cfg.initializer_range
+        self.is_vit_layer = is_vit_layer
+        nlayers = 1 if (is_vit_layer or config.vit_layers is None) else len(config.vit_layers)
+        self.wq = nn.Linear(
+            nlayers * self.embed_dim,
+            self.num_heads * self.head_dim,
+            bias=use_bias,
+            device=config.init_device,
+            )
+        self.wk = nn.Linear(
+            nlayers * self.embed_dim,
+            self.num_key_value_heads * self.head_dim,
+            bias=use_bias,
+            device=config.init_device,
+            )
+        self.wv = nn.Linear(
+            nlayers * self.embed_dim,
+            self.num_key_value_heads * self.head_dim,
+            bias=use_bias,
+            device=config.init_device,
+            )
+        self.wo = nn.Linear(
+            self.num_heads * self.head_dim,
+            self.embed_dim,
+            bias=use_bias,
+            device=config.init_device,
+            )
+        self.attention_dropout: Optional[Dropout] = None
+        if v_cfg.attention_dropout > 0:
+            self.attention_dropout = Dropout(v_cfg.attention_dropout, broadcast_dims=(0, 1))
+        self.residual_dropout = Dropout(v_cfg.residual_dropout)
+    def reset_parameters(self):
+        nn.init.normal_(self.wq.weight, std=self.initializer_range)
+        nn.init.normal_(self.wk.weight, std=self.initializer_range)
+        nn.init.normal_(self.wv.weight, std=self.initializer_range)
+        nn.init.normal_(self.wo.weight, std=self.initializer_range)
+        if self.use_bias:
+            nn.init.constant_(self.wq.bias, 0)
+            nn.init.constant_(self.wk.bias, 0)
+            nn.init.constant_(self.wv.bias, 0)
+            nn.init.constant_(self.wo.bias, 0)
+    def _split_heads(self, hidden_states, num_heads) -> torch.Tensor:
+        return hidden_states.reshape(hidden_states.shape[:2] + (num_heads, self.head_dim))
+    def _merge_heads(self, hidden_states) -> torch.Tensor:
+        return hidden_states.reshape(hidden_states.shape[:2] + (self.embed_dim,))
+    def forward(self, inputs_q: torch.Tensor, inputs_kv: Optional[torch.Tensor] = None) -> torch.Tensor:
+        if inputs_kv is not None:
+            inputs_k = inputs_kv
+            inputs_v = inputs_kv
+        else:
+            inputs_k = inputs_q
+            inputs_v = inputs_q
+        xq, xk, xv = self.wq(inputs_q), self.wk(inputs_k), self.wv(inputs_v)
+        xq = self._split_heads(xq, self.num_heads)
+        xk = self._split_heads(xk, self.num_key_value_heads)
+        xv = self._split_heads(xv, self.num_key_value_heads)
+        if self.num_heads != self.num_key_value_heads:
+            xk = xk.repeat_interleave(self.num_key_value_groups, dim=2, output_size=self.num_heads)
+            xv = xv.repeat_interleave(self.num_key_value_groups, dim=2, output_size=self.num_heads)
+        og_dtype = xq.dtype
+        if self.config.float32_attention:
+            xq = xq.to(torch.float)
+            xk = xk.to(torch.float)
+        if self.config.attention_type == "direct":
+            attn_weights = torch.einsum("...qhd,...khd->...hqk", xq / math.sqrt(xq.size(-1)), xk)
+            attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(xq.dtype)
+            if self.attention_dropout is not None:
+                attn_weights = self.attention_dropout(attn_weights)
+            attn_output = torch.einsum("...hqk,...khd->...qhd", attn_weights.to(xv.dtype), xv)
+        elif self.config.attention_type == "sdpa":
+            if self.config.float32_attention and not torch.is_autocast_enabled():
+                xv = xv.to(torch.float32)
+            attn_output = F.scaled_dot_product_attention(
+                xq.transpose(1, 2).contiguous(),
+                xk.transpose(1, 2).contiguous(),
+                xv.transpose(1, 2).contiguous(),
+                is_causal=False,
+                dropout_p=self.config.vision_backbone.attention_dropout
+            ).transpose(1, 2)
+        else:
+            raise NotImplementedError(self.config.attention_type)
+        attn_output = attn_output.to(og_dtype)
+        attn_output = self._merge_heads(attn_output)
+        attn_output = self.wo(attn_output)
+        attn_output = self.residual_dropout(attn_output)
+        return attn_output
+class MultiHeadAttentionPool(nn.Module):
+    def __init__(
+        self,
+        config: FullMolmoConfig,
+        factor: int = 1,
+        use_bias: bool = True,
+        dropout: bool = True,
+        output_layer: bool = True,
+        mean_residual: bool = False,
+        query: str = "mean",
+        is_vit_layer: Optional[bool] = True
+    ):
+        super().__init__()
+        self.config = config
+        self.factor = factor
+        self.use_bias = use_bias
+        self.dropout = dropout
+        self.output_layer = output_layer
+        self.mean_residual = mean_residual
+        self.query = query
+        v_cfg = config.vision_backbone
+        input_dim = v_cfg.image_emb_dim
+        self.embed_dim = v_cfg.image_emb_dim * factor
+        self.num_heads = v_cfg.image_num_heads
+        self.head_dim = v_cfg.image_head_dim * factor
+        self.num_key_value_heads = v_cfg.image_num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.initializer_range = v_cfg.initializer_range
+        nlayers = 1 if (is_vit_layer or config.vit_layers is None) else len(config.vit_layers)
+        if query != "vector":
+            self.wq = nn.Linear(
+                nlayers * input_dim,
+                self.num_heads * self.head_dim,
+                bias=use_bias,
+                device=config.init_device,
+                )
+        self.wk = nn.Linear(
+            nlayers * input_dim,
+            self.num_key_value_heads * self.head_dim,
+            bias=use_bias,
+            device=config.init_device,
+            )
+        self.wv = nn.Linear(
+            nlayers * input_dim,
+            self.num_key_value_heads * self.head_dim,
+            bias=use_bias,
+            device=config.init_device,
+            )
+        if query == "vector":
+            self.attention_query = nn.Parameter(
+                torch.zeros(
+                    1, self.num_key_value_heads * self.head_dim, device=config.init_device,
+                       ),
+            )
+        if output_layer:
+            self.wo = nn.Linear(
+                self.num_heads * self.head_dim,
+                self.embed_dim,
+                bias=use_bias,
+                device=config.init_device,
+                )
+        self.attention_dropout = Dropout(v_cfg.attention_dropout, broadcast_dims=(0, 1))
+        if dropout:
+            self.residual_dropout = Dropout(v_cfg.residual_dropout)
+    def reset_parameters(self):
+        if self.query != "vector":
+            nn.init.normal_(self.wq.weight, std=self.initializer_range)
+        nn.init.normal_(self.wk.weight, std=self.initializer_range)
+        nn.init.normal_(self.wv.weight, std=self.initializer_range)
+        if self.output_layer:
+            nn.init.normal_(self.wo.weight, std=self.initializer_range)
+        if self.use_bias:
+            if self.query != "vector":
+                nn.init.constant_(self.wq.bias, 0)
+            nn.init.constant_(self.wk.bias, 0)
+            nn.init.constant_(self.wv.bias, 0)
+            if self.output_layer:
+                nn.init.constant_(self.wo.bias, 0)
+        if self.query == "vector":
+            nn.init.normal_(self.attention_query, std=self.initializer_range)
+    def _split_heads(self, hidden_states, num_heads):
+        return hidden_states.reshape(hidden_states.shape[:2] + (num_heads, self.head_dim))
+    def _merge_heads(self, hidden_states):
+        return hidden_states.reshape(hidden_states.shape[:2] + (self.embed_dim,))
+    def forward(self, inputs_kv: torch.Tensor) -> torch.Tensor:
+        xk, xv = self.wk(inputs_kv), self.wv(inputs_kv)
+        if self.query == "mean":
+            inputs_q = inputs_kv.mean(dim=1, keepdim=True)
+            xq = self.wq(inputs_q)
+        elif self.query == "first":
+            inputs_q = inputs_kv[:, :1]
+            xq = self.wq(inputs_q)
+        elif self.query == "vector":
+            xq = self.attention_query.expand(inputs_kv.size(0), -1, -1)
+        elif self.query == "constant":
+            inputs_q = torch.ones_like(inputs_kv[:, :1]) / math.sqrt(inputs_kv.shape[-1])
+            xq = self.wq(inputs_q)
+        else:
+            raise ValueError(f"Unknown query type: {self.query}")
+        xq = self._split_heads(xq, self.num_heads)
+        xk = self._split_heads(xk, self.num_key_value_heads)
+        xv = self._split_heads(xv, self.num_key_value_heads)
+        if self.num_heads != self.num_key_value_heads:
+            xk = xk.repeat_interleave(self.num_key_value_groups, dim=2, output_size=self.num_heads)
+            xv = xv.repeat_interleave(self.num_key_value_groups, dim=2, output_size=self.num_heads)
+        xq = xq.to(torch.float)
+        xk = xk.to(torch.float)
+        xq = xq / math.sqrt(xq.size(-1))
+        attn_weights = torch.einsum("...qhd,...khd->...hqk", xq, xk)
+        attn_weights = F.softmax(attn_weights, dim=-1).to(xq.dtype)
+        attn_weights = self.attention_dropout(attn_weights).to(xv.dtype)
+        attn_output = torch.einsum("...hqk,...khd->...qhd", attn_weights, xv)
+        attn_output = self._merge_heads(attn_output)
+        if self.output_layer:
+            attn_output = self.wo(attn_output)
+        if self.dropout:
+            attn_output = self.residual_dropout(attn_output)
+        if self.mean_residual:
+            attn_output += inputs_kv.mean(dim=1, keepdim=True)
+        return attn_output
+class MLP(nn.Module):
+    def __init__(self, config: FullMolmoConfig, input_dim: int, dropout: float = 0.0):
+        super().__init__()
+        self.config = config
+        self.hidden_size = (
+            config.mlp_hidden_size if config.mlp_hidden_size is not None else config.mlp_ratio * config.d_model
+        )
+        self.initializer_range = config.initializer_range
+        self.w1 = nn.Linear(
+            input_dim,
+            self.hidden_size // 2,
+            bias=False,
+            device=config.init_device,
+            )
+        self.w2 = nn.Linear(
+            self.hidden_size // 2,
+            config.d_model,
+            bias=False,
+            device=config.init_device,
+            )
+        self.w3 = nn.Linear(
+            input_dim,
+            self.hidden_size // 2,
+            bias=False,
+            device=config.init_device,
+            )
+        # Activation function.
+        self.act = Activation.build(config)
+        self.dropout = Dropout(dropout)
+    def reset_parameters(self):
+        nn.init.normal_(self.w1.weight, std=self.initializer_range)
+        nn.init.normal_(self.w2.weight, std=self.initializer_range)
+        nn.init.normal_(self.w3.weight, std=self.initializer_range)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.w2(self.act(self.w1(x), self.w3(x)))
+        x = self.dropout(x)
+        return x
+class Residual(nn.Module):
+    def __init__(self, submodule: nn.Module):
+        super().__init__()
+        self.submodule = submodule
+    def reset_parameters(self):
+        self.submodule.reset_parameters()
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x + self.submodule(x)
+class OLMoVisionBackbone(nn.Module):
+    def __init__(self, config: FullMolmoConfig):
+        super().__init__()
+        self.config = config
+        self.image_vit = VisionTransformer(config)
+        input_dim: int = None
+        self.image_pooling_2d: nn.Module = None
+        if config.image_pooling_2d in {ImagePooling2DType.attention, ImagePooling2DType.attention_meanq}:
+            self.image_pooling_2d = MultiHeadDotProductAttention(config, is_vit_layer=False)
+            input_dim = config.vision_backbone.image_emb_dim
+        elif config.image_pooling_2d == ImagePooling2DType.attention_2wide:
+            cfg = deepcopy(config)
+            cfg.vision_backbone.image_emb_dim *= 2
+            cfg.vision_backbone.image_head_dim *= 2
+            self.image_pooling_2d = MultiHeadDotProductAttention(cfg, is_vit_layer=False)
+            input_dim = cfg.vision_backbone.image_emb_dim
+        elif config.image_pooling_2d == ImagePooling2DType.attention_v2:
+            assert config.vit_layers is not None
+            use_bias = True
+            dropout = True
+            output_layer = True
+            query = "mean"
+            mean_residual = False
+            factor = len(config.vit_layers)
+            self.image_pooling_2d = MultiHeadAttentionPool(
+                config,
+                factor=factor,
+                use_bias=use_bias,
+                dropout=dropout,
+                output_layer=output_layer,
+                mean_residual=mean_residual,
+                query=query,
+                is_vit_layer=False,
+            )
+            input_dim = config.vision_backbone.image_emb_dim * factor
+        elif config.image_pooling_2d in [ImagePooling2DType.none, ImagePooling2DType.stack]:
+            self.image_pooling_2d = None
+            nlayers = 1 if config.vit_layers is None else len(config.vit_layers)
+            input_dim = nlayers * config.vision_backbone.image_emb_dim
+        else:
+            raise NotImplementedError(f"Unknown image pooling 2D method: {config.image_pooling_2d}")
+        self.input_dim = input_dim
+        # `MLP` assume the activation takes two inputs, so it must be a 'llama' version
+        if config.activation_type == ActivationType.swiglu:
+            mlp_config = replace(config, activation_type=ActivationType.llama_swiglu)
+        elif config.activation_type == ActivationType.gelu:
+            mlp_config = replace(config, activation_type=ActivationType.llama_geglu)
+        else:
+            mlp_config = config
+        if config.image_projector == ImageProjectType.mlpx2:
+            self.image_projector = nn.ModuleList(
+                [MLP(mlp_config, input_dim), Residual(MLP(config, input_dim))]
+            )
+        elif config.image_projector == ImageProjectType.mlp:
+            self.image_projector = MLP(mlp_config, input_dim)
+        elif config.image_projector == ImageProjectType.linear:
+            self.image_projector = nn.Linear(
+                input_dim,
+                config.d_model,
+                bias=False,
+                device=config.init_device,
+            )
+        else:
+            raise NotImplementedError(f"Unknown image projector: {config.image_projector}")
+        self.image_feature_dropout = Dropout(config.image_feature_dropout)
+    def reset_parameters(self):
+        if self.image_pooling_2d is not None:
+            self.image_pooling_2d.reset_parameters()
+        if self.config.image_projector == "2mlp":
+            for module in self.image_projector:
+                module.reset_parameters()
+        elif self.config.image_projector == "linear":
+            nn.init.xavier_uniform_(self.image_projector.weight)
+        else:
+            self.image_projector.reset_parameters()
+    def forward(self, images: torch.Tensor, image_masks: torch.Tensor) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        raise NotImplementedError
+class OLMoPretrainedVisionBackbone(OLMoVisionBackbone):
+    def __init__(self, config: FullMolmoConfig):
+        super().__init__(config)
+        v_cfg = self.config.vision_backbone
+        self.grad_checkpointing = False
+        self.num_prefix_tokens = self.image_vit.num_prefix_tokens
+        assert self.num_prefix_tokens in {0, 1}, "Only 0 or 1 prefix tokens are supported"
+        self.pad_embed = None
+        if config.image_padding_embed:
+            image_dim = v_cfg.image_emb_dim*len(self.config.vit_layers)
+            if config.image_padding_embed in ["pad_embed", "regress"]:
+                self.pad_embed = nn.Parameter(
+                    torch.zeros((image_dim,), device=config.init_device))
+            elif config.image_padding_embed == "pad_and_partial_pad":
+                self.pad_embed = nn.Parameter(
+                    torch.zeros((2, image_dim), device=config.init_device))
+            else:
+                raise ValueError(config.image_padding_embed)
+    def reset_parameters(self):
+        super().reset_parameters()
+        self.image_vit.reset_parameters()
+    def encode_image(self, images: torch.Tensor) -> torch.Tensor:
+        """
+        : param images: (batch_size, num_crops, num_patch, n_pixels)
+        """
+        cfg = self.config
+        v_cfg = self.config.vision_backbone
+        B, T, N, D = images.shape
+        mask = ~torch.all(images.view(B * T, N, D) == -1, dim=(1, 2), keepdim=True)
+        # Output all hidden states
+        # n_layers x (batch_num_crops, (1+)n_tokens, image_emb_dim)
+        images = images.view(B * T, N, D)
+        image_features = self.image_vit(images)
+        if cfg.vit_layers is not None:
+            features = []
+            for layer in cfg.vit_layers:
+                features.append(image_features[layer])
+            image_features = torch.cat(features, dim=-1)
+        else:
+            image_features = image_features[-1]
+        cls_embed: torch.Tensor = None
+        if self.num_prefix_tokens > 0:
+            cls_embed = image_features[:, 0]
+            image_features = image_features[:, 1:]
+        image_features = image_features * mask
+        image_features = image_features.view(B, T, N, -1)
+        cls_embed = cls_embed.view(B, T, -1) if cls_embed is not None else None
+        return image_features, cls_embed
+    def forward(self, images: torch.Tensor, image_masks: torch.Tensor) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        cfg = self.config
+        # image_features: (batch_size, num_crops(=num_image), num_patch, nximage_emb_dim)
+        batch_size, num_image = images.shape[:2]
+        image_features, cls_embed = self.encode_image(images)
+        if cfg.image_padding_embed:
+            assert image_masks is not None
+            if cfg.image_padding_embed == "pad_embed":
+                all_pad = (image_masks == 0).to(dtype=torch.float32)
+                pad_embed = self.pad_embed[None, None, None, :]
+                image_features = image_features + pad_embed * torch.unsqueeze(all_pad, -1)
+            elif cfg.image_padding_embed == "regress":
+                pad_embed = self.pad_embed[None, None, None, :]
+                image_features = image_features + pad_embed * torch.unsqueeze(torch.maximum(image_masks, torch.zeros_like(image_masks)), -1)
+            elif cfg.image_padding_embed == "pad_and_partial_pad":
+                pad_embed = self.pad_embed[:, None, None, None, :]
+                all_pad = image_masks == 0
+                partial_pad = torch.logical_and(image_masks < 1, torch.logical_not(all_pad)).to(dtype=image_features.dtype)
+                all_pad = all_pad.to(dtype=image_features.dtype)
+                image_features = image_features + pad_embed[0] * torch.unsqueeze(all_pad, -1)
+                image_features = image_features + pad_embed[1] * torch.unsqueeze(partial_pad, -1)
+            else:
+                raise ValueError(cfg.image_padding_embed)
+        image_features = self.image_feature_dropout(image_features)
+        if cls_embed is not None:
+            cls_embed = self.image_feature_dropout(cls_embed)
+        image_features = image_features.reshape(
+            (batch_size, num_image) + cfg.image_num_patch + (-1,),
+            )
+        if cfg.image_num_patch[0] % cfg.image_pooling_h == 1:
+            # Pad so we can still pool 2x2 patches
+            image_features = F.pad(
+                image_features,
+                (0, 0, 0, 1, 0, 1, 0, 0, 0, 0),
+            )
+        # image pooling
+        image_features = einops.rearrange(
+            image_features,
+            'b n (h dh) (w dw) c -> (b n h w) (dh dw) c',
+            dh=cfg.image_pooling_h,
+            dw=cfg.image_pooling_w,
+        )
+        if cfg.image_pooling_2d == ImagePooling2DType.attention_meanq:
+            query = image_features.mean(-2, keepdim=True)
+            image_features = self.image_pooling_2d(query, image_features)
+        elif cfg.image_pooling_2d not in {ImagePooling2DType.none, ImagePooling2DType.stack}:
+            if self.grad_checkpointing:
+                from torch.utils.checkpoint import checkpoint
+                image_features = checkpoint(self.image_pooling_2d, image_features[:, :1, :], image_features, use_reentrant=False)
+            else:
+                image_features = self.image_pooling_2d(image_features[:, :1, :], image_features)
+        h, w = cfg.llm_patches_per_crop()
+        image_features = image_features.reshape(batch_size, num_image, h * w, -1)
+        # MLP layer to map the feature.
+        if self.grad_checkpointing:
+            from torch.utils.checkpoint import checkpoint
+            image_features = checkpoint(self.image_projector, image_features, use_reentrant=False)
+        else:
+            image_features = self.image_projector(image_features)
+        # image_features: (batch_size, num_image, num_patch, d_model)
+        # cls_embed: (batch_size, num_image, d_model)
+        return image_features, cls_embed
+class ModuleType(str, Enum):
+    in_module = "in"
+    out_module = "out"
+    emb = "emb"
+    final_out = "final_out"
+def init_weights(
+    config: FullMolmoConfig,
+    module: Union[nn.Linear, nn.Embedding],
+    d: Optional[int] = None,
+    layer_id: Optional[int] = None,
+    std_factor: float = 1.0,
+    type_of_module: Optional[ModuleType] = None,
+) -> None:
+    d = d if d is not None else config.d_model
+    std = config.init_std * std_factor
+    if config.init_cutoff_factor is not None:
+        cutoff_value = config.init_cutoff_factor * std
+        nn.init.trunc_normal_(module.weight, mean=0.0, std=std, a=-cutoff_value, b=cutoff_value)
+    else:
+        nn.init.normal_(module.weight, mean=0.0, std=std)
+class LlamaSwiGLU(nn.Module):
+    def forward(self, x1: torch.Tensor, x2: torch.Tensor) -> torch.Tensor:
+        return F.silu(x1) * x2
+    @property
+    def output_multiplier(self) -> float:
+        return 0.5
+class SwiGLU(nn.Module):
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+    @property
+    def output_multiplier(self) -> float:
+        return 0.5
+class Activation(nn.Module):
+    def __init__(self, config: FullMolmoConfig):
+        super().__init__()
+        self.config = config
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        raise NotImplementedError
+    @property
+    def output_multiplier(self) -> float:
+        raise NotImplementedError
+    @classmethod
+    def build(cls, config: FullMolmoConfig) -> 'Activation':
+        if config.activation_type == "quick_gelu":
+            return QuickGELU(config)
+        elif config.activation_type == "gelu":
+            return cast(Activation, GELU(approximate="none"))
+        elif config.activation_type == "gelu_tanh":
+            return cast(Activation, GELU(approximate="tanh"))
+        elif config.activation_type == "relu":
+            return cast(Activation, ReLU(inplace=False))
+        elif config.activation_type == "silu":
+            return cast(Activation, SiLU(inplace=False))
+        # elif config.activation_type == "llama_geglu":
+        #     return LlamaGEGLU(config)
+        # elif config.activation_type == "llama_geglu_tanh":
+        #     return LlamaGEGLUTanh(config)
+        elif config.activation_type == "llama_swiglu":
+            return LlamaSwiGLU()
+        elif config.activation_type == "swiglu":
+            return SwiGLU()
+        else:
+            raise NotImplementedError(f"Unknown activation: '{config.activation_type}'")
+class QuickGELU(Activation):
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x * torch.sigmoid(1.702 * x)
+    @property
+    def output_multiplier(self) -> float:
+        return 1.0
+class GELU(nn.GELU):
+    @property
+    def output_multiplier(self) -> float:
+        return 1.0
+class ReLU(nn.ReLU):
+    @property
+    def output_multiplier(self) -> float:
+        return 1.0
+class SiLU(nn.SiLU):
+    @property
+    def output_multiplier(self) -> float:
+        return 1.0
+def causal_attention_bias(seq_len: int, device: torch.device) -> torch.FloatTensor:
+    att_bias = torch.triu(
+        torch.ones(seq_len, seq_len, device=device, dtype=torch.float),
+        diagonal=1,
+    )
+    att_bias.masked_fill_(att_bias == 1, torch.finfo(att_bias.dtype).min)
+    return att_bias.view(1, 1, seq_len, seq_len)  # type: ignore
+def get_causal_attention_bias(cache: BufferCache, seq_len: int, device: torch.device) -> torch.Tensor:
+    if (causal_bias := cache.get("causal_attention_bias")) is not None and causal_bias.shape[-1] >= seq_len:
+        if causal_bias.device != device:
+            causal_bias = causal_bias.to(device)
+            cache["causal_attention_bias"] = causal_bias
+        return causal_bias
+    with torch.autocast(device.type, enabled=False):
+        causal_bias = causal_attention_bias(seq_len, device)
+    cache["causal_attention_bias"] = causal_bias
+    return causal_bias
+class LayerNormBase(nn.Module):
+    def __init__(
+        self,
+        config: MolmoConfig,
+        *,
+        size: Optional[int] = None,
+        elementwise_affine: Optional[bool] = True,
+        eps: float = 1e-05,
+        weight_initializer: Optional[Callable] = torch.ones,
+        bias_initializer: Optional[Callable] = torch.zeros,
+    ):
+        super().__init__()
+        self.config = config
+        self.eps = self.config.layer_norm_eps or eps
+        self.normalized_shape = (size or config.d_model,)
+        if elementwise_affine or (elementwise_affine is None and self.config.layer_norm_with_affine):
+            self.weight = nn.Parameter(weight_initializer(self.normalized_shape, device=config.init_device))
+            use_bias = self.config.bias_for_layer_norm
+            if use_bias is None:
+                use_bias = self.config.include_bias
+            if use_bias:
+                self.bias = nn.Parameter(bias_initializer(self.normalized_shape, device=config.init_device))
+            else:
+                self.register_parameter("bias", None)
+        else:
+            self.register_parameter("bias", None)
+            self.register_parameter("weight", None)
+    @classmethod
+    def build(cls, config: FullMolmoConfig, size: Optional[int] = None, **kwargs):
+        if config.layer_norm_type == "default":
+            return LayerNorm(config, size=size, low_precision=False, **kwargs)
+        elif config.layer_norm_type == "low_precision":
+            return LayerNorm(config, size=size, low_precision=True, **kwargs)
+        elif config.layer_norm_type == "rms":
+            return RMSLayerNorm(config, size=size, **kwargs)
+        else:
+            raise NotImplementedError(f"Unknown LayerNorm type: '{config.layer_norm_type}'")
+class RMSLayerNorm(LayerNormBase):
+    """
+    RMS layer norm, a simplified :class:`LayerNorm` implementation
+    """
+    def __init__(
+        self,
+        config: FullMolmoConfig,
+        size: Optional[int] = None,
+        elementwise_affine: Optional[bool] = None,
+        eps: float = 1e-5,
+    ):
+        super().__init__(config, size=size, elementwise_affine=elementwise_affine, eps=eps)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        with torch.autocast(enabled=False, device_type=x.device.type):
+            og_dtype = x.dtype
+            x = x.to(torch.float32)
+            variance = x.pow(2).mean(-1, keepdim=True)
+            x = x * torch.rsqrt(variance + self.eps)
+            x = x.to(og_dtype)
+        if self.weight is not None:
+            if self.bias is not None:
+                return self.weight * x + self.bias
+            else:
+                return self.weight * x
+        else:
+            return x
+class LayerNorm(LayerNormBase):
+    """
+    The default :class:`LayerNorm` implementation which can optionally run in low precision.
+    """
+    def __init__(
+        self,
+        config: FullMolmoConfig,
+        size: Optional[int] = None,
+        low_precision: bool = False,
+        elementwise_affine: Optional[bool] = None,
+        eps: float = 1e-05,
+    ):
+        super().__init__(config, size=size, elementwise_affine=elementwise_affine, eps=eps)
+        self.low_precision = low_precision
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if self.low_precision:
+            module_device = x.device
+            downcast_x = self._cast_if_autocast_enabled(x)
+            downcast_weight = (
+                self._cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight
+            )
+            downcast_bias = self._cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias
+            with torch.autocast(enabled=False, device_type=module_device.type):
+                return F.layer_norm(
+                    downcast_x, self.normalized_shape, weight=downcast_weight, bias=downcast_bias, eps=self.eps
+                )
+        else:
+            return F.layer_norm(x, self.normalized_shape, weight=self.weight, bias=self.bias, eps=self.eps)
+class Molmo(nn.Module):
+    def __init__(self, config: FullMolmoConfig, init_params: bool = True):
+        super().__init__()
+        self.config = config
+        self.__cache = BufferCache()
+        # Validate config.
+        if self.config.embedding_size is not None and self.config.embedding_size != self.config.vocab_size:
+            if self.config.embedding_size < self.config.vocab_size:
+                raise MolmoConfigurationError("embedding size should be at least as big as vocab size")
+            elif self.config.embedding_size % 128 != 0:
+                import warnings
+                warnings.warn(
+                    "Embedding size is not a multiple of 128! This could hurt throughput performance.", UserWarning
+                )
+        torch.backends.cuda.enable_flash_sdp(True)
+        torch.backends.cuda.enable_mem_efficient_sdp(False)  # this is super slow so make sure torch won't use it
+        wte = None
+        if self.config.additional_vocab_size is not None:
+            wte = Embedding(
+                config.embedding_size or config.vocab_size,
+                config.additional_vocab_size,
+                config.d_model,
+                device=config.init_device,
+                initializer_range=config.initializer_range,
+                new_embed_initializer_range=config.new_embedding_init_range
+            )
+        else:
+            wte=nn.Embedding(
+                config.embedding_size or config.vocab_size, config.d_model, device=config.init_device
+            )
+        self.transformer = nn.ModuleDict(
+            dict(
+                wte=wte,
+                emb_drop=Dropout(config.embedding_dropout),
+                ln_f=LayerNorm.build(config),
+            )
+        )
+        blocks = [MolmoBlock.build(i, config, self.__cache) for i in range(config.n_layers)]
+        if self.config.block_group_size > 1:
+            raise NotImplementedError()
+        else:
+            self.transformer.update({"blocks": nn.ModuleList(blocks)})
+        if not self.config.rope:
+            self.transformer.update(
+                {"wpe": nn.Embedding(config.max_sequence_length, config.d_model, device=config.init_device)}
+            )
+        if not config.weight_tying:
+            self.transformer.update(
+                {
+                    "ff_out": nn.Linear(
+                        config.d_model,
+                        config.embedding_size or config.vocab_size,
+                        bias=config.include_bias,
+                        device=config.init_device,
+                        )
+                }
+            )
+        self.vision_backbone: Optional[OLMoVisionBackbone] = None
+        if config.vision_backbone is not None:
+            self.vision_backbone = OLMoPretrainedVisionBackbone(config)
+        self.__num_fwd_flops: Optional[int] = None
+    def reset_parameters(self):
+        if self.vision_backbone is not None:
+            self.vision_backbone.reset_parameters()
+        self.reset_non_vision_parameters()
+    def reset_non_vision_parameters(self):
+        self.transformer.wte.reset_parameters()
+        if hasattr(self.transformer.wte, "new_embedding"):
+            nn.init.normal_(self.transformer.wte.new_embedding, std=self.config.new_embedding_init_range)
+        if hasattr(self.transformer, "wpe"):
+            nn.init.normal_(self.transformer.wpe, mean=0.0, std=1.0)
+        self.transformer.ln_f.reset_parameters()  # type: ignore
+        if hasattr(self.transformer, "ff_out"):
+            nn.init.normal_(self.transformer.ff_out, mean=0.0, std=0.02)
+        if self.config.block_group_size == 1:
+            for block in self.transformer.blocks:
+                block.reset_parameters()
+        else:
+            for block_group in self.transformer.block_groups:
+                block_group.reset_parameters()
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        input_embeddings: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        attention_bias: Optional[torch.Tensor] = None,
+        response_mask: Optional[torch.Tensor] = None,
+        images: Optional[torch.Tensor] = None,
+        image_masks: Optional[torch.Tensor] = None,
+        image_input_idx: Optional[torch.Tensor] = None,
+        subsegment_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Sequence[Tuple[torch.Tensor, torch.Tensor]]] = None,
+        use_cache: bool = False,
+        last_logits_only: bool = False,
+        output_hidden_states: Optional[bool] = None,
+        append_last_valid_logits: Optional[torch.Tensor] = None,
+    ) -> ModelOutput:
+        """
+        :param input_ids: A tensor of shape `(batch_size, seq_len)`.
+        :param input_embeddings: A tensor of shape `(batch_size, seq_len, d_model)` with input
+            embeddings. When provided, it is treated as the output of the input embedding layer.
+        :param attention_mask: A tensor of shape `(batch_size, seq_len)` that indicates
+            which input IDs are masked. A `1` value in the mask means that
+            the corresponding input ID should *not* be ignored. A `0` means
+            that the corresponding input ID is masked.
+            This has the same meaning as the `attention_mask` in HuggingFace's `transformers`
+            library.
+        :param attention_bias: A tensor of shape `(batch_size, 1, seq_len, seq_len)`,
+            `(1, 1, seq_len, seq_len)`, or `(seq_len, seq_len)`. This is used
+            to introduce causal or other biases.
+            If the tensor is a bool or byte tensor, a `True` or `1` at `attention_bias[:, :, i, j]`
+            indicates that the i-th element in the sequence is allowed to attend to the j-th
+            element in the sequence.
+            If the tensor is a float tensor, it will just be added to the attention
+            scores before the softmax.
+            The default is causal, which corresponds to a lower-diagonal byte matrix of ones.
+        :param response_mask: A tensor of shape `(batch_size, seq_len)` that indicates
+            the response mask. A `1` value in the mask means that the corresponding token
+            is a response token. A `0` means that the corresponding token is not
+            a response token.
+        :param past_key_values: Pre-computed keys and values for each attention block.
+            Can be used to speed up sequential decoding. The `input_ids` which have
+            their past given to this model should not be passed as `input_ids` as they have already been computed.
+        :param use_cache: If `True`, return key and value tensors for each block.
+        :param last_logits_only: If `True`, only compute the logits for the last token of each sequence.
+            This can speed up decoding when you only care about the next token.
+        """
+        output_hidden_states = output_hidden_states if output_hidden_states is not None else False
+        if past_key_values:
+            assert len(past_key_values) == self.config.n_layers
+        has_image = images is not None
+        assert not (has_image and input_embeddings is not None), "Cannot provide both images and input embeddings."
+        assert not (has_image and past_key_values is not None), "Cached key and values should not be used with images."
+        batch_size, seq_len = input_ids.size() if input_embeddings is None else input_embeddings.size()[:2]
+        if past_key_values is None:
+            past_length = 0
+        else:
+            past_length = past_key_values[0][0].size(-2)
+        if self.config.use_position_ids and attention_mask is None:
+            attention_mask = input_ids != -1
+        if subsegment_ids is not None:
+            assert not use_cache, "Subsegment_ids cannot be used with cache."
+            subsegment_mask = subsegment_ids.unsqueeze(2) <= subsegment_ids.unsqueeze(1)
+            attention_mask = (
+                subsegment_mask.to(attention_mask.dtype) *
+                attention_mask.unsqueeze(2) *
+                attention_mask.unsqueeze(1))
+            if position_ids is None:
+                raise ValueError(f"Positioned ids must be given if using subsegment_ids")
+        else:
+            if self.config.use_position_ids and position_ids is None:
+                position_ids = torch.clamp(
+                    torch.cumsum(attention_mask.to(torch.int32), dim=-1) - 1,
+                    min=0,
+                    ).broadcast_to((batch_size, attention_mask.shape[-1]))
+        # Get embeddings of input.
+        # shape: (batch_size, seq_len, d_model)
+        if input_ids is not None:
+            input_ids = input_ids * (input_ids != -1).to(input_ids.dtype)
+        x = self.transformer.wte(input_ids) if input_embeddings is None else input_embeddings  # type: ignore
+        num_image: Optional[int] = None
+        if images is not None:
+            # shape: (batch_size, num_image, num_patch, d_model)
+            # cls_embed: (batch_size, num_image, d_model)
+            image_features, cls_embed = self.vision_backbone(images, image_masks)
+            num_image, num_patch = image_features.shape[1:3]
+            assert image_input_idx.shape == (batch_size, num_image, num_patch)
+            # inster the image feature into the embedding.
+            image_features = image_features.view(batch_size, num_image * num_patch, -1)
+            image_input_idx = image_input_idx.view(batch_size, num_image * num_patch)
+            valid = image_input_idx >= 0
+            batch_idx = torch.arange(batch_size, device=x.device)
+            batch_idx = torch.tile(batch_idx[:, None], [1, image_features.shape[1]])
+            # For hf demo/endpoint
+            image_features = image_features.to(x.device)
+            x[batch_idx[valid], image_input_idx[valid]] += image_features[valid]
+        if not self.config.rope:
+            # Get positional embeddings.
+            # shape: (1, seq_len)
+            pos = torch.arange(past_length, past_length + seq_len, dtype=torch.long, device=x.device).unsqueeze(0)
+            # shape: (1, seq_len, d_model)
+            pos_emb = self.transformer.wpe(pos)  # type: ignore
+            x = pos_emb + x
+        # Add input + positional embeddings and apply dropout.
+        # shape: (batch_size, seq_len, d_model)
+        x = self.transformer.emb_drop(x)  # type: ignore
+        # normalized
+        if self.config.normalize_input_embeds:
+            x = x * (self.config.d_model ** 0.5)
+        # Transform the attention mask into what the blocks expect.
+        if attention_mask is not None:
+            # shape: (batch_size, 1, 1, seq_len)
+            if len(attention_mask.shape) == 2:
+                attention_mask = attention_mask[:, :past_length + seq_len]
+                attention_mask = attention_mask.to(dtype=torch.float).view(batch_size, -1)[:, None, None, :]
+            else:
+                attention_mask = attention_mask.unsqueeze(1).to(dtype=torch.float)
+            attention_mask = (1.0 - attention_mask) * torch.finfo(attention_mask.dtype).min
+        # Merge attention mask with attention bias.
+        if (
+            attention_bias is not None
+            or attention_mask is not None
+            # NOTE (epwalsh): we need to initialize the attn bias in order for attn to work properly
+            # with key+value cache. Otherwise `F.scaled_dot_product_attention()` doesn't seem to compute
+            # scores correctly.
+            or past_key_values is not None
+        ):
+            if attention_bias is None:
+                attention_bias = get_causal_attention_bias(self.__cache, past_length + seq_len, x.device)
+            elif attention_bias.dtype in (torch.int8, torch.bool):
+                attention_bias = attention_bias.to(dtype=torch.float)
+                attention_bias.masked_fill_(attention_bias == 0.0, torch.finfo(attention_bias.dtype).min)
+            # Transform to the right shape and data type.
+            mask_len = seq_len
+            if attention_mask is not None:
+                mask_len = attention_mask.shape[-1]
+            elif past_key_values is not None:
+                mask_len = past_key_values[0][0].shape[-2] + seq_len
+            attention_bias = attention_bias[:, :, :mask_len, :mask_len].to(dtype=torch.float)
+            # Add in the masking bias.
+            if attention_mask is not None:
+                attention_bias = attention_bias + attention_mask
+                # Might get -infs after adding attention mask, since dtype.min + dtype.min = -inf.
+                # `F.scaled_dot_product_attention()` doesn't handle -inf like you'd expect, instead
+                # it can produce NaNs.
+                ensure_finite_(attention_bias, check_neg_inf=True, check_pos_inf=False)
+        attn_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = [] if use_cache else None
+        # decoder layers
+        all_hidden_states = []
+        # Apply blocks one-by-one.
+        if self.config.block_group_size == 1:
+            for block_idx, block in enumerate(self.transformer.blocks):
+                if output_hidden_states:
+                    # add hidden states
+                    all_hidden_states.append(x)
+                layer_past = None if past_key_values is None else past_key_values[block_idx]
+                x, cache = block(x, attention_bias=attention_bias, position_ids=position_ids, layer_past=layer_past, use_cache=use_cache)
+                if attn_key_values is not None:
+                    assert cache is not None
+                    attn_key_values.append(cache)
+        else:
+            for group_idx, block_group in enumerate(self.transformer.block_groups):
+                if output_hidden_states:
+                    # add hidden states
+                    all_hidden_states.append(x)
+                layers_past = (
+                    None
+                    if past_key_values is None
+                    else past_key_values[
+                         group_idx * self.config.block_group_size : (group_idx + 1) * self.config.block_group_size
+                         ]
+                )
+                x, cache = block_group(
+                    x, attention_bias=attention_bias, position_ids=position_ids, layers_past=layers_past, use_cache=use_cache
+                )
+                if attn_key_values is not None:
+                    assert cache is not None
+                    attn_key_values.extend(cache)
+        if last_logits_only:
+            # shape: (batch_size, 1, d_model)
+            if append_last_valid_logits is not None:
+                last_valid_output = x[
+                    torch.arange(x.shape[0], device=x.device), append_last_valid_logits.to(x.device)]
+                x = last_valid_output.unsqueeze(1)
+            else:
+                x = x[:, -1, :].unsqueeze(1)
+        # Apply final layer norm.
+        # shape: (batch_size, seq_len or 1, d_model)
+        x = self.transformer.ln_f(x)  # type: ignore
+        if output_hidden_states:
+            # add final hidden state post-final-layernorm, following HuggingFace's convention
+            all_hidden_states.append(x)
+        # Get logits.
+        # shape: (batch_size, seq_len or 1, vocab_size)
+        if self.config.weight_tying:
+            logits = F.linear(x, self.transformer.wte.weight, None)  # type: ignore
+        else:
+            logits = self.transformer.ff_out(x)  # type: ignore
+        if self.config.scale_logits:
+            logits.mul_(1 / math.sqrt(self.config.d_model))
+        if not last_logits_only and append_last_valid_logits is not None:
+            last_valid_logit = logits[
+                torch.arange(logits.shape[0], device=logits.device), append_last_valid_logits]
+            logits = torch.cat([logits[:, :-1], last_valid_logit[:, None]], dim=1)
+        return ModelOutput(logits=logits, attn_key_values=attn_key_values, hidden_states=tuple(all_hidden_states) if output_hidden_states else None)  # type: ignore[arg-type]
+class MolmoForCausalLM(PreTrainedModel):
+    config_class = MolmoConfig
+    base_model_prefix = "model"
+    _no_split_modules = ["MolmoBlock"]
+    def __init__(self, config: MolmoConfig, model: Optional[Molmo] = None, init_params: bool = False):
+        super().__init__(config)
+        if not model:
+            full_config = FullMolmoConfig(
+                image_padding_embed="pad_and_partial_pad",
+                image_pooling_2d="attention-meanq",
+                attention_layer_norm=config.attention_layer_norm,
+                rope_impl="llama",
+                vocab_size=config.vocab_size,
+                max_sequence_length=config.max_position_embeddings,
+                qkv_bias=config.qkv_bias,
+                norm_after=config.norm_after,
+                embedding_size=config.embedding_size,
+                attention_type="sdpa",
+                embedding_dropout=0,
+                attention_dropout=0,
+                residual_dropout=0,
+                rope=True,
+                weight_tying=False,
+                include_bias=False,
+                d_model=config.hidden_size,
+                mlp_hidden_size=config.intermediate_size,
+                n_layers=config.num_hidden_layers,
+                additional_vocab_size=128,
+                n_heads=config.num_attention_heads,
+                n_kv_heads=config.num_key_value_heads,
+                rope_theta=config.rope_theta,
+                layer_norm_eps=config.layer_norm_eps,
+                layer_norm_type=config.layer_norm_type,
+                vit_layers=[-2, -9],
+                vision_backbone=VisionBackboneConfig(
+                    image_default_input_size=(336, 336),
+                    image_patch_size=14,
+                    image_pos_patch_size=14,
+                    image_emb_dim=1024,
+                    image_num_heads=16,
+                    image_num_key_value_heads=16,
+                    image_num_layers=23,
+                    image_head_dim=64,
+                    image_mlp_dim=4096,
+                    image_mlp_activations="quick_gelu",
+                    image_dropout_rate=0.0,
+                    image_num_pos=577,
+                    image_norm_eps=1e-5,
+                    attention_dropout=0.0,
+                    residual_dropout=0.0,
+                    initializer_range=0.02,
+                )
+            )
+            self.model = Molmo(full_config, init_params=init_params)
+        else:
+            self.model = model
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        attention_bias: Optional[torch.Tensor] = None,
+        response_mask: Optional[torch.Tensor] = None,
+        images: Optional[torch.Tensor] = None,
+        image_masks: Optional[torch.Tensor] = None,
+        image_input_idx: Optional[torch.Tensor] = None,
+        subsegment_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        labels: Optional[torch.LongTensor] = None,
+        loss_masks: Optional[torch.Tensor] = None,
+        use_cache: Optional[bool] = None,
+        last_logits_only: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        append_last_valid_logits: Optional[torch.Tensor] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[
+            Cache
+        ] = None,  # This is a hack mitigation of an issue in transformers `4.39.x` https://github.com/huggingface/transformers/issues/29426
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        if use_cache is None:
+            use_cache = self.config.use_cache
+        if output_attentions:
+            raise ValueError("output_attentions is not yet supported in Molmo")
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model.forward(
+            input_ids=input_ids,
+            input_embeddings=inputs_embeds,
+            attention_mask=attention_mask,
+            attention_bias=attention_bias,
+            response_mask=response_mask,
+            images=images,
+            image_masks=image_masks,
+            image_input_idx=image_input_idx,
+            subsegment_ids=subsegment_ids,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            last_logits_only=last_logits_only,
+            output_hidden_states=output_hidden_states,
+            append_last_valid_logits=append_last_valid_logits,
+        )
+        logits = outputs.logits
+        hidden_states = outputs.hidden_states
+        loss = None
+        if labels is not None:
+            if loss_masks is not None:
+                loss_masks = loss_masks * (loss_masks > 0)
+                batch_size_in_tokens = max(loss_masks.sum().item(), 1)
+                labels = labels.long()
+                labels.masked_fill_(~(loss_masks > 0), -100)
+                labels = labels.view(-1)
+                logits_for_loss = logits.to(torch.float32).view(-1, logits.size(-1))
+                loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100, reduction='none')
+                loss = loss_fct(logits_for_loss, labels)
+                loss = loss.view(input_ids.shape[0], -1)
+                loss = loss * loss_masks
+                loss = loss.sum() / batch_size_in_tokens
+                use_zloss = getattr(self.config, "softmax_auxiliary_loss", False)
+                if use_zloss:
+                    z_squared = logits_for_loss.logsumexp(-1).pow(2)
+                    z_loss = self.config.softmax_auxiliary_loss_scale * z_squared
+                    z_loss = z_loss.view(input_ids.shape[0], -1)
+                    z_loss = z_loss * loss_masks
+                    z_loss = z_loss.sum() / batch_size_in_tokens
+                    loss += z_loss
+            else:
+                # Shift so that tokens < n predict n
+                shift_logits = logits[..., :-1, :].contiguous()
+                shift_labels = labels[..., 1:].contiguous()
+                # Flatten the tokens
+                loss_fct = torch.nn.CrossEntropyLoss()
+                shift_logits = shift_logits.view(-1, self.config.embedding_size)
+                shift_labels = shift_labels.view(-1)
+                # Enable model parallelism
+                shift_labels = shift_labels.to(shift_logits.device)
+                loss = loss_fct(shift_logits, shift_labels)
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.attn_key_values,
+            hidden_states=hidden_states,
+        )
+    def can_generate(self) -> bool:
+        return True
+    @torch.no_grad()
+    def generate_from_batch(
+        self,
+        batch: Dict[str, Any],
+        generation_config: Optional[GenerationConfig] = None,
+        **kwargs,
+    ):
+        if generation_config is not None:
+            assert generation_config.use_cache
+        images = batch.get("images")
+        image_masks = batch.get("image_masks")
+        image_input_idx = batch.get("image_input_idx")
+        # Validate inputs.
+        input_ids = batch["input_ids"]
+        batch_size, seq_len = input_ids.shape
+        attention_mask = batch.get("attention_mask", None)
+        max_new_tokens = generation_config.max_new_tokens
+        assert max_new_tokens is not None
+        mask_len = seq_len + max_new_tokens if self.config.use_position_ids else seq_len
+        position_ids: Optional[torch.Tensor] = None
+        append_last_valid_logits: Optional[torch.Tensor] = None
+        if self.config.use_position_ids and attention_mask is None:
+            attention_mask = input_ids != -1
+            position_ids = torch.clamp(
+                torch.cumsum(attention_mask.to(torch.int32), dim=-1) - 1,
+                min=0
+            )
+            append_last_valid_logits = attention_mask.long().sum(dim=-1) - 1
+            attention_mask = torch.cat(
+                [attention_mask, attention_mask.new_ones((batch_size, max_new_tokens))],
+                dim=1,
+            )
+        if attention_mask is not None:
+            assert attention_mask.shape == (batch_size, mask_len)
+        out = super().generate(
+            batch["input_ids"],
+            generation_config,
+            attention_mask=attention_mask,
+            images=images,
+            image_masks=image_masks,
+            image_input_idx=image_input_idx,
+            position_ids=position_ids,
+            append_last_valid_logits=append_last_valid_logits,
+            **kwargs,
+        )
+        return out
+    def prepare_inputs_for_generation(
+        self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple]] = None, **kwargs
+    ):
+        if past_key_values:
+            # This is because we want the model to only process the last generated token.
+            input_ids = input_ids[:, -1:]
+        if self.config.use_position_ids:
+            attention_mask = kwargs.get("attention_mask")
+            images = kwargs.get("images")
+            image_masks = kwargs.get("image_masks")
+            image_input_idx = kwargs.get("image_input_idx")
+            position_ids = kwargs.get("position_ids")
+            append_last_valid_logits = kwargs.get("append_last_valid_logits")
+            model_inputs = {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": True,
+                "last_logits_only": True,
+            }
+            if past_key_values is None:
+                model_inputs["images"] = images
+                model_inputs["image_masks"] = image_masks
+                model_inputs["image_input_idx"] = image_input_idx
+                model_inputs["append_last_valid_logits"] = append_last_valid_logits
+        else:
+            model_inputs = {"input_ids": input_ids, "past_key_values": past_key_values}
+            model_inputs.update(kwargs)
+            model_inputs["use_cache"] = kwargs.pop("use_cache", self.config.use_cache)
+        return model_inputs
+    def _update_model_kwargs_for_generation(
+        self,
+        outputs: ModelOutput,
+        model_kwargs: Dict[str, Any],
+        is_encoder_decoder: bool = False,
+        num_new_tokens: int = 1,
+    ) -> Dict[str, Any]:
+        if self.config.use_position_ids:
+            model_kwargs["position_ids"] = model_kwargs["position_ids"][:, -1:] + 1
+            if "append_last_valid_logits" in model_kwargs:
+                del model_kwargs["append_last_valid_logits"]
+            if "images" in model_kwargs:
+                del model_kwargs["images"]
+                del model_kwargs["image_masks"]
+                del model_kwargs["image_input_idx"]
+        try:
+            cache_name, cache = super()._extract_past_from_model_output(outputs)
+        except AttributeError:
+            past_key_values = outputs.past_key_values if "past_key_values" in outputs else None
+            cache_name, cache = "past_key_values", past_key_values
+        model_kwargs[cache_name] = cache
+        model_kwargs["cache_position"] = model_kwargs["cache_position"][-1:] + num_new_tokens
+        return model_kwargs
+    def get_input_embeddings(self) -> torch.nn.Module:
+        return self.model.transformer.wte
+    def set_input_embeddings(self, value: torch.nn.Module):
+        self.model.transformer.wte = value
+    def get_output_embeddings(self):
+        if self.config.weight_tying:
+            return self.model.transformer.wte
+        else:
+            return self.model.transformer.ff_out
+    def set_output_embeddings(self, value: torch.nn.Module):
+        if self.config.weight_tying:
+            self.model.transformer.wte = value
+        else:
+            self.model.transformer.ff_out = value
+    def tie_weights(self):
+        """
+        This function is intentionally left as a no-op.
+        Weight tying is handled as follows:
+        - When the model is initialized, the `ff_out` layer is conditionally defined based on the `weight_tying` configuration.
+        See: `if not config.weight_tying: self.transformer.update(...)` in `olmo/model.py`.
+        - When computing logits, the `wte` weights are used directly if `weight_tying` is enabled.
+        See: `if self.config.weight_tying: logits = F.linear(x, self.transformer.wte.weight, None)` in the `forward` method.
+        Therefore, there is no need to explicitly tie the weights in this function.
+        """
+        pass
+    def resize_token_embeddings(
+        self, new_num_tokens: Optional[int] = None, pad_to_multiple_of: Optional[int] = None
+    ) -> torch.nn.Embedding:
+        """
+        Resizes input token embeddings matrix of the model if `new_num_tokens != config.embedding_size`.
+        Takes care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.
+        Arguments:
+            new_num_tokens (`int`, *optional*):
+                The new number of tokens in the embedding matrix. Increasing the size will add newly initialized
+                vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just
+                returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing anything.
+            pad_to_multiple_of (`int`, *optional*):
+                If set will pad the embedding matrix to a multiple of the provided value. If `new_num_tokens` is set to
+                `None` will just pad the embedding to a multiple of `pad_to_multiple_of`.
+                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
+                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more
+                details about this, or help on choosing the correct value for resizing, refer to this guide:
+                https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc
+        Return:
+            `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model.
+        Note:
+            This method differs from the base class implementation by resizing the `embedding_size` attribute of the
+            model configuration instead of the `vocab_size`. It also includes a warning if the resized `embedding_size`
+            is less than the `vocab_size`. In OLMo, `embedding_size` refers to the dimensionality of the model's token
+            embeddings, while `vocab_size` refers to the number of unique tokens in the vocabulary.
+        """
+        model_embeds = self._resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+        if new_num_tokens is None and pad_to_multiple_of is None:
+            return model_embeds
+        # Update base model and current model config
+        self.config.embedding_size = model_embeds.weight.shape[0]
+        self.model.config.embedding_size = model_embeds.weight.shape[0]
+        # Check if the embedding size is less than the vocab size
+        if self.config.embedding_size < self.config.vocab_size:
+            warning_message = (
+                f"Resizing token embeddings to size {self.config.embedding_size}, which is less than the vocab size "
+                f"{self.config.vocab_size} defined in the model configuration. Make sure your tokenizer's vocabulary "
+                "size is less than or equal to the new token embedding size."
+            )
+            log.warning(warning_message)
+        # Tie weights again if needed
+        self.tie_weights()
+        return model_embeds
+# Always register for multi-modal features
+AutoModelForCausalLM.register(MolmoConfig, MolmoForCausalLM)

preprocessing_molmo.py ADDED Viewed

	@@ -0,0 +1,192 @@

+"""
+Processor class for Molmo.
+"""
+from typing import Optional
+import PIL
+from PIL import ImageOps
+from PIL.Image import Image
+try:
+    from typing import Unpack
+except ImportError:
+    from typing_extensions import Unpack
+import numpy as np
+import torch
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import (
+    TextKwargs,
+    ProcessingKwargs,
+    ProcessorMixin,
+)
+from transformers.tokenization_utils_base import TextInput, PreTokenizedInput
+from transformers.utils import logging
+from transformers import AutoTokenizer
+from .image_preprocessing_molmo import MolmoImagesKwargs, MolmoImageProcessor
+logger = logging.get_logger(__name__)
+DEFAULT_IMAGE_PATCH_TOKEN = f"<im_patch>"
+DEFAULT_IM_START_TOKEN = f"<im_start>"
+DEFAULT_IM_END_TOKEN = f"<im_end>"
+DEFAULT_IM_COL_TOKEN = f"<im_col>"
+IMAGE_PROMPT = "<|image|>"
+EXTRA_TOKENS = (DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_COL_TOKEN, IMAGE_PROMPT)
+def get_special_token_ids(tokenizer):
+    ids = tokenizer.encode("".join(EXTRA_TOKENS), add_special_tokens=False)
+    assert len(ids) == len(EXTRA_TOKENS)
+    return {k: i for k, i in zip(EXTRA_TOKENS, ids)}
+class MolmoTextKwargs(TextKwargs, total=False):
+    style: Optional[str]
+    system_prompt: Optional[str]
+    message_format: Optional[str]
+    always_start_with_space: Optional[bool]
+    sequence_length: Optional[int]
+class MolmoProcessorKwargs(ProcessingKwargs, total=False):
+    text_kwargs: MolmoTextKwargs
+    images_kwargs: MolmoImagesKwargs
+    _defaults = {
+        "images_kwargs": {
+            "max_crops": 12,
+            "overlap_margins": [4, 4],
+            "base_image_input_size": [336, 336],
+            "image_token_length_w": 12,
+            "image_token_length_h": 12,
+            "image_patch_size": 14,
+            "image_padding_mask": True,
+        },
+        "text_kwargs": {
+            "style": "long_caption",
+            "system_prompt": "none",
+            "message_format": "role",
+            "always_start_with_space": True,
+            "sequence_length": 1536,
+            "padding": False,
+        },
+    }
+class MolmoProcessor(ProcessorMixin):
+    attributes = ["image_processor", "tokenizer"]
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
+    def __init__(self, image_processor: MolmoImageProcessor = None, tokenizer : AutoTokenizer = None, **kwargs):
+        # self.image_processor = image_processor
+        # self.tokenizer = tokenizer
+        super().__init__(image_processor, tokenizer)
+        self._special_tokens = None
+    @property
+    def special_token_ids(self):
+        if self._special_tokens is None:
+            self._special_tokens = get_special_token_ids(self.tokenizer)
+        return self._special_tokens
+    def get_tokens_input(self, prompt, message_format, always_start_with_space):
+        if message_format == "none" or message_format is None:
+            pass
+        elif message_format == "role":
+            prompt = "User: " + prompt + " Assistant:"
+        else:
+            raise NotImplementedError(f"Message format {message_format} not implemented")
+        if always_start_with_space:
+            prompt = " " + prompt
+        tokens = self.tokenizer.encode(prompt, add_special_tokens=False)
+        return tokens
+    def process(
+        self,
+        text: TextInput = None,
+        images: ImageInput = None,
+        *,
+        tokens: Optional[PreTokenizedInput] = None,
+        **kwargs: Unpack[MolmoProcessorKwargs],
+    ):
+        output_kwargs = self._merge_kwargs(
+            MolmoProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        if tokens is None:
+            tokens = self.get_tokens_input(
+                text,
+                output_kwargs["text_kwargs"]["message_format"],
+                output_kwargs["text_kwargs"]["always_start_with_space"],
+            )
+        image_token_id = self.special_token_ids[IMAGE_PROMPT]
+        if images is not None:
+            if not isinstance(images, (list, tuple)):
+                images = [images]
+            image_arrays = []
+            for image in images:
+                if isinstance(image, Image):
+                    image = image.convert("RGB")
+                    # Handle images with EXIF orientation tags, which PIL will ignore by default
+                    # https://github.com/python-pillow/Pillow/issues/4703
+                    img = ImageOps.exif_transpose(image)
+                    image_arrays.append(np.array(image))
+                else:
+                    assert len(image.shape) == 3 and image.shape[-1] == 3
+                    image_arrays.append(image.astype(np.uint8))
+            images = image_arrays
+            # For now only support inserting images at the start
+            image_idx = [-1]*len(images)
+        else:
+            image_idx = None
+        sequence_length = output_kwargs["text_kwargs"]["sequence_length"]
+        image_patch_token_id = self.special_token_ids[DEFAULT_IMAGE_PATCH_TOKEN]
+        image_col_token_id = self.special_token_ids[DEFAULT_IM_COL_TOKEN]
+        image_start_token_id = self.special_token_ids[DEFAULT_IM_START_TOKEN]
+        image_end_token_id = self.special_token_ids[DEFAULT_IM_END_TOKEN]
+        out = self.image_processor.multimodal_preprocess(
+            images=images,
+            image_idx=image_idx,
+            tokens=np.asarray(tokens).astype(np.int32),
+            sequence_length=sequence_length,
+            image_patch_token_id=image_patch_token_id,
+            image_col_token_id=image_col_token_id,
+            image_start_token_id=image_start_token_id,
+            image_end_token_id=image_end_token_id,
+            **output_kwargs["images_kwargs"]
+        )
+        # Prepend BOS
+        # qwen2 and olmo do not have a BOS, and instead use EOS as a generic seperator token.
+        bos = self.tokenizer.bos_token_id or self.tokenizer.eos_token_id
+        decoder_input_tokens = np.pad(out["input_ids"], [[1, 0]], constant_values=bos)
+        out["input_ids"] = decoder_input_tokens
+        if "image_input_idx" in out:
+            # Shift patch mapping up by one since we added BOS
+            image_input_idx = out["image_input_idx"]
+            out["image_input_idx"] = np.where(image_input_idx < 0, image_input_idx, image_input_idx + 1)
+        for k, v in out.items():
+            out[k] = torch.from_numpy(v)
+        return out
+MolmoProcessor.register_for_auto_class()

preprocessor_config.json ADDED Viewed

	@@ -0,0 +1,32 @@

+{
+  "auto_map": {
+    "AutoImageProcessor": "image_preprocessing_molmo.MolmoImageProcessor",
+    "AutoProcessor": "preprocessing_molmo.MolmoProcessor"
+  },
+  "base_image_input_size": [
+    336,
+    336
+  ],
+  "do_normalize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_padding_mask": true,
+  "image_patch_size": 14,
+  "image_processor_type": "MolmoImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "image_token_length_h": 12,
+  "image_token_length_w": 12,
+  "max_crops": 12,
+  "overlap_margins": [
+    4,
+    4
+  ],
+  "processor_class": "MolmoProcessor"
+}

processor_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "auto_map": {
+    "AutoProcessor": "preprocessing_molmo.MolmoProcessor"
+  },
+  "processor_class": "MolmoProcessor"
+}

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,2073 @@

+{
+  "additional_special_tokens": [
+    "<soa>",
+    "<eoa>",
+    "<audio_pad>",
+    "<audio_0>",
+    "<audio_1>",
+    "<audio_2>",
+    "<audio_3>",
+    "<audio_4>",
+    "<audio_5>",
+    "<audio_6>",
+    "<audio_7>",
+    "<audio_8>",
+    "<audio_9>",
+    "<audio_10>",
+    "<audio_11>",
+    "<audio_12>",
+    "<audio_13>",
+    "<audio_14>",
+    "<audio_15>",
+    "<audio_16>",
+    "<audio_17>",
+    "<audio_18>",
+    "<audio_19>",
+    "<audio_20>",
+    "<audio_21>",
+    "<audio_22>",
+    "<audio_23>",
+    "<audio_24>",
+    "<audio_25>",
+    "<audio_26>",
+    "<audio_27>",
+    "<audio_28>",
+    "<audio_29>",
+    "<audio_30>",
+    "<audio_31>",
+    "<audio_32>",
+    "<audio_33>",
+    "<audio_34>",
+    "<audio_35>",
+    "<audio_36>",
+    "<audio_37>",
+    "<audio_38>",
+    "<audio_39>",
+    "<audio_40>",
+    "<audio_41>",
+    "<audio_42>",
+    "<audio_43>",
+    "<audio_44>",
+    "<audio_45>",
+    "<audio_46>",
+    "<audio_47>",
+    "<audio_48>",
+    "<audio_49>",
+    "<audio_50>",
+    "<audio_51>",
+    "<audio_52>",
+    "<audio_53>",
+    "<audio_54>",
+    "<audio_55>",
+    "<audio_56>",
+    "<audio_57>",
+    "<audio_58>",
+    "<audio_59>",
+    "<audio_60>",
+    "<audio_61>",
+    "<audio_62>",
+    "<audio_63>",
+    "<audio_64>",
+    "<audio_65>",
+    "<audio_66>",
+    "<audio_67>",
+    "<audio_68>",
+    "<audio_69>",
+    "<audio_70>",
+    "<audio_71>",
+    "<audio_72>",
+    "<audio_73>",
+    "<audio_74>",
+    "<audio_75>",
+    "<audio_76>",
+    "<audio_77>",
+    "<audio_78>",
+    "<audio_79>",
+    "<audio_80>",
+    "<audio_81>",
+    "<audio_82>",
+    "<audio_83>",
+    "<audio_84>",
+    "<audio_85>",
+    "<audio_86>",
+    "<audio_87>",
+    "<audio_88>",
+    "<audio_89>",
+    "<audio_90>",
+    "<audio_91>",
+    "<audio_92>",
+    "<audio_93>",
+    "<audio_94>",
+    "<audio_95>",
+    "<audio_96>",
+    "<audio_97>",
+    "<audio_98>",
+    "<audio_99>",
+    "<audio_100>",
+    "<audio_101>",
+    "<audio_102>",
+    "<audio_103>",
+    "<audio_104>",
+    "<audio_105>",
+    "<audio_106>",
+    "<audio_107>",
+    "<audio_108>",
+    "<audio_109>",
+    "<audio_110>",
+    "<audio_111>",
+    "<audio_112>",
+    "<audio_113>",
+    "<audio_114>",
+    "<audio_115>",
+    "<audio_116>",
+    "<audio_117>",
+    "<audio_118>",
+    "<audio_119>",
+    "<audio_120>",
+    "<audio_121>",
+    "<audio_122>",
+    "<audio_123>",
+    "<audio_124>",
+    "<audio_125>",
+    "<audio_126>",
+    "<audio_127>",
+    "<audio_128>",
+    "<audio_129>",
+    "<audio_130>",
+    "<audio_131>",
+    "<audio_132>",
+    "<audio_133>",
+    "<audio_134>",
+    "<audio_135>",
+    "<audio_136>",
+    "<audio_137>",
+    "<audio_138>",
+    "<audio_139>",
+    "<audio_140>",
+    "<audio_141>",
+    "<audio_142>",
+    "<audio_143>",
+    "<audio_144>",
+    "<audio_145>",
+    "<audio_146>",
+    "<audio_147>",
+    "<audio_148>",
+    "<audio_149>",
+    "<audio_150>",
+    "<audio_151>",
+    "<audio_152>",
+    "<audio_153>",
+    "<audio_154>",
+    "<audio_155>",
+    "<audio_156>",
+    "<audio_157>",
+    "<audio_158>",
+    "<audio_159>",
+    "<audio_160>",
+    "<audio_161>",
+    "<audio_162>",
+    "<audio_163>",
+    "<audio_164>",
+    "<audio_165>",
+    "<audio_166>",
+    "<audio_167>",
+    "<audio_168>",
+    "<audio_169>",
+    "<audio_170>",
+    "<audio_171>",
+    "<audio_172>",
+    "<audio_173>",
+    "<audio_174>",
+    "<audio_175>",
+    "<audio_176>",
+    "<audio_177>",
+    "<audio_178>",
+    "<audio_179>",
+    "<audio_180>",
+    "<audio_181>",
+    "<audio_182>",
+    "<audio_183>",
+    "<audio_184>",
+    "<audio_185>",
+    "<audio_186>",
+    "<audio_187>",
+    "<audio_188>",
+    "<audio_189>",
+    "<audio_190>",
+    "<audio_191>",
+    "<audio_192>",
+    "<audio_193>",
+    "<audio_194>",
+    "<audio_195>",
+    "<audio_196>",
+    "<audio_197>",
+    "<audio_198>",
+    "<audio_199>",
+    "<audio_200>",
+    "<audio_201>",
+    "<audio_202>",
+    "<audio_203>",
+    "<audio_204>",
+    "<audio_205>",
+    "<audio_206>",
+    "<audio_207>",
+    "<audio_208>",
+    "<audio_209>",
+    "<audio_210>",
+    "<audio_211>",
+    "<audio_212>",
+    "<audio_213>",
+    "<audio_214>",
+    "<audio_215>",
+    "<audio_216>",
+    "<audio_217>",
+    "<audio_218>",
+    "<audio_219>",
+    "<audio_220>",
+    "<audio_221>",
+    "<audio_222>",
+    "<audio_223>",
+    "<audio_224>",
+    "<audio_225>",
+    "<audio_226>",
+    "<audio_227>",
+    "<audio_228>",
+    "<audio_229>",
+    "<audio_230>",
+    "<audio_231>",
+    "<audio_232>",
+    "<audio_233>",
+    "<audio_234>",
+    "<audio_235>",
+    "<audio_236>",
+    "<audio_237>",
+    "<audio_238>",
+    "<audio_239>",
+    "<audio_240>",
+    "<audio_241>",
+    "<audio_242>",
+    "<audio_243>",
+    "<audio_244>",
+    "<audio_245>",
+    "<audio_246>",
+    "<audio_247>",
+    "<audio_248>",
+    "<audio_249>",
+    "<audio_250>",
+    "<audio_251>",
+    "<audio_252>",
+    "<audio_253>",
+    "<audio_254>",
+    "<audio_255>",
+    "<audio_256>",
+    "<audio_257>",
+    "<audio_258>",
+    "<audio_259>",
+    "<audio_260>",
+    "<audio_261>",
+    "<audio_262>",
+    "<audio_263>",
+    "<audio_264>",
+    "<audio_265>",
+    "<audio_266>",
+    "<audio_267>",
+    "<audio_268>",
+    "<audio_269>",
+    "<audio_270>",
+    "<audio_271>",
+    "<audio_272>",
+    "<audio_273>",
+    "<audio_274>",
+    "<audio_275>",
+    "<audio_276>",
+    "<audio_277>",
+    "<audio_278>",
+    "<audio_279>",
+    "<audio_280>",
+    "<audio_281>",
+    "<audio_282>",
+    "<audio_283>",
+    "<audio_284>",
+    "<audio_285>",
+    "<audio_286>",
+    "<audio_287>",
+    "<audio_288>",
+    "<audio_289>",
+    "<audio_290>",
+    "<audio_291>",
+    "<audio_292>",
+    "<audio_293>",
+    "<audio_294>",
+    "<audio_295>",
+    "<audio_296>",
+    "<audio_297>",
+    "<audio_298>",
+    "<audio_299>",
+    "<audio_300>",
+    "<audio_301>",
+    "<audio_302>",
+    "<audio_303>",
+    "<audio_304>",
+    "<audio_305>",
+    "<audio_306>",
+    "<audio_307>",
+    "<audio_308>",
+    "<audio_309>",
+    "<audio_310>",
+    "<audio_311>",
+    "<audio_312>",
+    "<audio_313>",
+    "<audio_314>",
+    "<audio_315>",
+    "<audio_316>",
+    "<audio_317>",
+    "<audio_318>",
+    "<audio_319>",
+    "<audio_320>",
+    "<audio_321>",
+    "<audio_322>",
+    "<audio_323>",
+    "<audio_324>",
+    "<audio_325>",
+    "<audio_326>",
+    "<audio_327>",
+    "<audio_328>",
+    "<audio_329>",
+    "<audio_330>",
+    "<audio_331>",
+    "<audio_332>",
+    "<audio_333>",
+    "<audio_334>",
+    "<audio_335>",
+    "<audio_336>",
+    "<audio_337>",
+    "<audio_338>",
+    "<audio_339>",
+    "<audio_340>",
+    "<audio_341>",
+    "<audio_342>",
+    "<audio_343>",
+    "<audio_344>",
+    "<audio_345>",
+    "<audio_346>",
+    "<audio_347>",
+    "<audio_348>",
+    "<audio_349>",
+    "<audio_350>",
+    "<audio_351>",
+    "<audio_352>",
+    "<audio_353>",
+    "<audio_354>",
+    "<audio_355>",
+    "<audio_356>",
+    "<audio_357>",
+    "<audio_358>",
+    "<audio_359>",
+    "<audio_360>",
+    "<audio_361>",
+    "<audio_362>",
+    "<audio_363>",
+    "<audio_364>",
+    "<audio_365>",
+    "<audio_366>",
+    "<audio_367>",
+    "<audio_368>",
+    "<audio_369>",
+    "<audio_370>",
+    "<audio_371>",
+    "<audio_372>",
+    "<audio_373>",
+    "<audio_374>",
+    "<audio_375>",
+    "<audio_376>",
+    "<audio_377>",
+    "<audio_378>",
+    "<audio_379>",
+    "<audio_380>",
+    "<audio_381>",
+    "<audio_382>",
+    "<audio_383>",
+    "<audio_384>",
+    "<audio_385>",
+    "<audio_386>",
+    "<audio_387>",
+    "<audio_388>",
+    "<audio_389>",
+    "<audio_390>",
+    "<audio_391>",
+    "<audio_392>",
+    "<audio_393>",
+    "<audio_394>",
+    "<audio_395>",
+    "<audio_396>",
+    "<audio_397>",
+    "<audio_398>",
+    "<audio_399>",
+    "<audio_400>",
+    "<audio_401>",
+    "<audio_402>",
+    "<audio_403>",
+    "<audio_404>",
+    "<audio_405>",
+    "<audio_406>",
+    "<audio_407>",
+    "<audio_408>",
+    "<audio_409>",
+    "<audio_410>",
+    "<audio_411>",
+    "<audio_412>",
+    "<audio_413>",
+    "<audio_414>",
+    "<audio_415>",
+    "<audio_416>",
+    "<audio_417>",
+    "<audio_418>",
+    "<audio_419>",
+    "<audio_420>",
+    "<audio_421>",
+    "<audio_422>",
+    "<audio_423>",
+    "<audio_424>",
+    "<audio_425>",
+    "<audio_426>",
+    "<audio_427>",
+    "<audio_428>",
+    "<audio_429>",
+    "<audio_430>",
+    "<audio_431>",
+    "<audio_432>",
+    "<audio_433>",
+    "<audio_434>",
+    "<audio_435>",
+    "<audio_436>",
+    "<audio_437>",
+    "<audio_438>",
+    "<audio_439>",
+    "<audio_440>",
+    "<audio_441>",
+    "<audio_442>",
+    "<audio_443>",
+    "<audio_444>",
+    "<audio_445>",
+    "<audio_446>",
+    "<audio_447>",
+    "<audio_448>",
+    "<audio_449>",
+    "<audio_450>",
+    "<audio_451>",
+    "<audio_452>",
+    "<audio_453>",
+    "<audio_454>",
+    "<audio_455>",
+    "<audio_456>",
+    "<audio_457>",
+    "<audio_458>",
+    "<audio_459>",
+    "<audio_460>",
+    "<audio_461>",
+    "<audio_462>",
+    "<audio_463>",
+    "<audio_464>",
+    "<audio_465>",
+    "<audio_466>",
+    "<audio_467>",
+    "<audio_468>",
+    "<audio_469>",
+    "<audio_470>",
+    "<audio_471>",
+    "<audio_472>",
+    "<audio_473>",
+    "<audio_474>",
+    "<audio_475>",
+    "<audio_476>",
+    "<audio_477>",
+    "<audio_478>",
+    "<audio_479>",
+    "<audio_480>",
+    "<audio_481>",
+    "<audio_482>",
+    "<audio_483>",
+    "<audio_484>",
+    "<audio_485>",
+    "<audio_486>",
+    "<audio_487>",
+    "<audio_488>",
+    "<audio_489>",
+    "<audio_490>",
+    "<audio_491>",
+    "<audio_492>",
+    "<audio_493>",
+    "<audio_494>",
+    "<audio_495>",
+    "<audio_496>",
+    "<audio_497>",
+    "<audio_498>",
+    "<audio_499>",
+    "<audio_500>",
+    "<audio_501>",
+    "<audio_502>",
+    "<audio_503>",
+    "<audio_504>",
+    "<audio_505>",
+    "<audio_506>",
+    "<audio_507>",
+    "<audio_508>",
+    "<audio_509>",
+    "<audio_510>",
+    "<audio_511>",
+    "<audio_512>",
+    "<audio_513>",
+    "<audio_514>",
+    "<audio_515>",
+    "<audio_516>",
+    "<audio_517>",
+    "<audio_518>",
+    "<audio_519>",
+    "<audio_520>",
+    "<audio_521>",
+    "<audio_522>",
+    "<audio_523>",
+    "<audio_524>",
+    "<audio_525>",
+    "<audio_526>",
+    "<audio_527>",
+    "<audio_528>",
+    "<audio_529>",
+    "<audio_530>",
+    "<audio_531>",
+    "<audio_532>",
+    "<audio_533>",
+    "<audio_534>",
+    "<audio_535>",
+    "<audio_536>",
+    "<audio_537>",
+    "<audio_538>",
+    "<audio_539>",
+    "<audio_540>",
+    "<audio_541>",
+    "<audio_542>",
+    "<audio_543>",
+    "<audio_544>",
+    "<audio_545>",
+    "<audio_546>",
+    "<audio_547>",
+    "<audio_548>",
+    "<audio_549>",
+    "<audio_550>",
+    "<audio_551>",
+    "<audio_552>",
+    "<audio_553>",
+    "<audio_554>",
+    "<audio_555>",
+    "<audio_556>",
+    "<audio_557>",
+    "<audio_558>",
+    "<audio_559>",
+    "<audio_560>",
+    "<audio_561>",
+    "<audio_562>",
+    "<audio_563>",
+    "<audio_564>",
+    "<audio_565>",
+    "<audio_566>",
+    "<audio_567>",
+    "<audio_568>",
+    "<audio_569>",
+    "<audio_570>",
+    "<audio_571>",
+    "<audio_572>",
+    "<audio_573>",
+    "<audio_574>",
+    "<audio_575>",
+    "<audio_576>",
+    "<audio_577>",
+    "<audio_578>",
+    "<audio_579>",
+    "<audio_580>",
+    "<audio_581>",
+    "<audio_582>",
+    "<audio_583>",
+    "<audio_584>",
+    "<audio_585>",
+    "<audio_586>",
+    "<audio_587>",
+    "<audio_588>",
+    "<audio_589>",
+    "<audio_590>",
+    "<audio_591>",
+    "<audio_592>",
+    "<audio_593>",
+    "<audio_594>",
+    "<audio_595>",
+    "<audio_596>",
+    "<audio_597>",
+    "<audio_598>",
+    "<audio_599>",
+    "<audio_600>",
+    "<audio_601>",
+    "<audio_602>",
+    "<audio_603>",
+    "<audio_604>",
+    "<audio_605>",
+    "<audio_606>",
+    "<audio_607>",
+    "<audio_608>",
+    "<audio_609>",
+    "<audio_610>",
+    "<audio_611>",
+    "<audio_612>",
+    "<audio_613>",
+    "<audio_614>",
+    "<audio_615>",
+    "<audio_616>",
+    "<audio_617>",
+    "<audio_618>",
+    "<audio_619>",
+    "<audio_620>",
+    "<audio_621>",
+    "<audio_622>",
+    "<audio_623>",
+    "<audio_624>",
+    "<audio_625>",
+    "<audio_626>",
+    "<audio_627>",
+    "<audio_628>",
+    "<audio_629>",
+    "<audio_630>",
+    "<audio_631>",
+    "<audio_632>",
+    "<audio_633>",
+    "<audio_634>",
+    "<audio_635>",
+    "<audio_636>",
+    "<audio_637>",
+    "<audio_638>",
+    "<audio_639>",
+    "<audio_640>",
+    "<audio_641>",
+    "<audio_642>",
+    "<audio_643>",
+    "<audio_644>",
+    "<audio_645>",
+    "<audio_646>",
+    "<audio_647>",
+    "<audio_648>",
+    "<audio_649>",
+    "<audio_650>",
+    "<audio_651>",
+    "<audio_652>",
+    "<audio_653>",
+    "<audio_654>",
+    "<audio_655>",
+    "<audio_656>",
+    "<audio_657>",
+    "<audio_658>",
+    "<audio_659>",
+    "<audio_660>",
+    "<audio_661>",
+    "<audio_662>",
+    "<audio_663>",
+    "<audio_664>",
+    "<audio_665>",
+    "<audio_666>",
+    "<audio_667>",
+    "<audio_668>",
+    "<audio_669>",
+    "<audio_670>",
+    "<audio_671>",
+    "<audio_672>",
+    "<audio_673>",
+    "<audio_674>",
+    "<audio_675>",
+    "<audio_676>",
+    "<audio_677>",
+    "<audio_678>",
+    "<audio_679>",
+    "<audio_680>",
+    "<audio_681>",
+    "<audio_682>",
+    "<audio_683>",
+    "<audio_684>",
+    "<audio_685>",
+    "<audio_686>",
+    "<audio_687>",
+    "<audio_688>",
+    "<audio_689>",
+    "<audio_690>",
+    "<audio_691>",
+    "<audio_692>",
+    "<audio_693>",
+    "<audio_694>",
+    "<audio_695>",
+    "<audio_696>",
+    "<audio_697>",
+    "<audio_698>",
+    "<audio_699>",
+    "<audio_700>",
+    "<audio_701>",
+    "<audio_702>",
+    "<audio_703>",
+    "<audio_704>",
+    "<audio_705>",
+    "<audio_706>",
+    "<audio_707>",
+    "<audio_708>",
+    "<audio_709>",
+    "<audio_710>",
+    "<audio_711>",
+    "<audio_712>",
+    "<audio_713>",
+    "<audio_714>",
+    "<audio_715>",
+    "<audio_716>",
+    "<audio_717>",
+    "<audio_718>",
+    "<audio_719>",
+    "<audio_720>",
+    "<audio_721>",
+    "<audio_722>",
+    "<audio_723>",
+    "<audio_724>",
+    "<audio_725>",
+    "<audio_726>",
+    "<audio_727>",
+    "<audio_728>",
+    "<audio_729>",
+    "<audio_730>",
+    "<audio_731>",
+    "<audio_732>",
+    "<audio_733>",
+    "<audio_734>",
+    "<audio_735>",
+    "<audio_736>",
+    "<audio_737>",
+    "<audio_738>",
+    "<audio_739>",
+    "<audio_740>",
+    "<audio_741>",
+    "<audio_742>",
+    "<audio_743>",
+    "<audio_744>",
+    "<audio_745>",
+    "<audio_746>",
+    "<audio_747>",
+    "<audio_748>",
+    "<audio_749>",
+    "<audio_750>",
+    "<audio_751>",
+    "<audio_752>",
+    "<audio_753>",
+    "<audio_754>",
+    "<audio_755>",
+    "<audio_756>",
+    "<audio_757>",
+    "<audio_758>",
+    "<audio_759>",
+    "<audio_760>",
+    "<audio_761>",
+    "<audio_762>",
+    "<audio_763>",
+    "<audio_764>",
+    "<audio_765>",
+    "<audio_766>",
+    "<audio_767>",
+    "<audio_768>",
+    "<audio_769>",
+    "<audio_770>",
+    "<audio_771>",
+    "<audio_772>",
+    "<audio_773>",
+    "<audio_774>",
+    "<audio_775>",
+    "<audio_776>",
+    "<audio_777>",
+    "<audio_778>",
+    "<audio_779>",
+    "<audio_780>",
+    "<audio_781>",
+    "<audio_782>",
+    "<audio_783>",
+    "<audio_784>",
+    "<audio_785>",
+    "<audio_786>",
+    "<audio_787>",
+    "<audio_788>",
+    "<audio_789>",
+    "<audio_790>",
+    "<audio_791>",
+    "<audio_792>",
+    "<audio_793>",
+    "<audio_794>",
+    "<audio_795>",
+    "<audio_796>",
+    "<audio_797>",
+    "<audio_798>",
+    "<audio_799>",
+    "<audio_800>",
+    "<audio_801>",
+    "<audio_802>",
+    "<audio_803>",
+    "<audio_804>",
+    "<audio_805>",
+    "<audio_806>",
+    "<audio_807>",
+    "<audio_808>",
+    "<audio_809>",
+    "<audio_810>",
+    "<audio_811>",
+    "<audio_812>",
+    "<audio_813>",
+    "<audio_814>",
+    "<audio_815>",
+    "<audio_816>",
+    "<audio_817>",
+    "<audio_818>",
+    "<audio_819>",
+    "<audio_820>",
+    "<audio_821>",
+    "<audio_822>",
+    "<audio_823>",
+    "<audio_824>",
+    "<audio_825>",
+    "<audio_826>",
+    "<audio_827>",
+    "<audio_828>",
+    "<audio_829>",
+    "<audio_830>",
+    "<audio_831>",
+    "<audio_832>",
+    "<audio_833>",
+    "<audio_834>",
+    "<audio_835>",
+    "<audio_836>",
+    "<audio_837>",
+    "<audio_838>",
+    "<audio_839>",
+    "<audio_840>",
+    "<audio_841>",
+    "<audio_842>",
+    "<audio_843>",
+    "<audio_844>",
+    "<audio_845>",
+    "<audio_846>",
+    "<audio_847>",
+    "<audio_848>",
+    "<audio_849>",
+    "<audio_850>",
+    "<audio_851>",
+    "<audio_852>",
+    "<audio_853>",
+    "<audio_854>",
+    "<audio_855>",
+    "<audio_856>",
+    "<audio_857>",
+    "<audio_858>",
+    "<audio_859>",
+    "<audio_860>",
+    "<audio_861>",
+    "<audio_862>",
+    "<audio_863>",
+    "<audio_864>",
+    "<audio_865>",
+    "<audio_866>",
+    "<audio_867>",
+    "<audio_868>",
+    "<audio_869>",
+    "<audio_870>",
+    "<audio_871>",
+    "<audio_872>",
+    "<audio_873>",
+    "<audio_874>",
+    "<audio_875>",
+    "<audio_876>",
+    "<audio_877>",
+    "<audio_878>",
+    "<audio_879>",
+    "<audio_880>",
+    "<audio_881>",
+    "<audio_882>",
+    "<audio_883>",
+    "<audio_884>",
+    "<audio_885>",
+    "<audio_886>",
+    "<audio_887>",
+    "<audio_888>",
+    "<audio_889>",
+    "<audio_890>",
+    "<audio_891>",
+    "<audio_892>",
+    "<audio_893>",
+    "<audio_894>",
+    "<audio_895>",
+    "<audio_896>",
+    "<audio_897>",
+    "<audio_898>",
+    "<audio_899>",
+    "<audio_900>",
+    "<audio_901>",
+    "<audio_902>",
+    "<audio_903>",
+    "<audio_904>",
+    "<audio_905>",
+    "<audio_906>",
+    "<audio_907>",
+    "<audio_908>",
+    "<audio_909>",
+    "<audio_910>",
+    "<audio_911>",
+    "<audio_912>",
+    "<audio_913>",
+    "<audio_914>",
+    "<audio_915>",
+    "<audio_916>",
+    "<audio_917>",
+    "<audio_918>",
+    "<audio_919>",
+    "<audio_920>",
+    "<audio_921>",
+    "<audio_922>",
+    "<audio_923>",
+    "<audio_924>",
+    "<audio_925>",
+    "<audio_926>",
+    "<audio_927>",
+    "<audio_928>",
+    "<audio_929>",
+    "<audio_930>",
+    "<audio_931>",
+    "<audio_932>",
+    "<audio_933>",
+    "<audio_934>",
+    "<audio_935>",
+    "<audio_936>",
+    "<audio_937>",
+    "<audio_938>",
+    "<audio_939>",
+    "<audio_940>",
+    "<audio_941>",
+    "<audio_942>",
+    "<audio_943>",
+    "<audio_944>",
+    "<audio_945>",
+    "<audio_946>",
+    "<audio_947>",
+    "<audio_948>",
+    "<audio_949>",
+    "<audio_950>",
+    "<audio_951>",
+    "<audio_952>",
+    "<audio_953>",
+    "<audio_954>",
+    "<audio_955>",
+    "<audio_956>",
+    "<audio_957>",
+    "<audio_958>",
+    "<audio_959>",
+    "<audio_960>",
+    "<audio_961>",
+    "<audio_962>",
+    "<audio_963>",
+    "<audio_964>",
+    "<audio_965>",
+    "<audio_966>",
+    "<audio_967>",
+    "<audio_968>",
+    "<audio_969>",
+    "<audio_970>",
+    "<audio_971>",
+    "<audio_972>",
+    "<audio_973>",
+    "<audio_974>",
+    "<audio_975>",
+    "<audio_976>",
+    "<audio_977>",
+    "<audio_978>",
+    "<audio_979>",
+    "<audio_980>",
+    "<audio_981>",
+    "<audio_982>",
+    "<audio_983>",
+    "<audio_984>",
+    "<audio_985>",
+    "<audio_986>",
+    "<audio_987>",
+    "<audio_988>",
+    "<audio_989>",
+    "<audio_990>",
+    "<audio_991>",
+    "<audio_992>",
+    "<audio_993>",
+    "<audio_994>",
+    "<audio_995>",
+    "<audio_996>",
+    "<audio_997>",
+    "<audio_998>",
+    "<audio_999>",
+    "<audio_1000>",
+    "<audio_1001>",
+    "<audio_1002>",
+    "<audio_1003>",
+    "<audio_1004>",
+    "<audio_1005>",
+    "<audio_1006>",
+    "<audio_1007>",
+    "<audio_1008>",
+    "<audio_1009>",
+    "<audio_1010>",
+    "<audio_1011>",
+    "<audio_1012>",
+    "<audio_1013>",
+    "<audio_1014>",
+    "<audio_1015>",
+    "<audio_1016>",
+    "<audio_1017>",
+    "<audio_1018>",
+    "<audio_1019>",
+    "<audio_1020>",
+    "<audio_1021>",
+    "<audio_1022>",
+    "<audio_1023>",
+    "<audio_1024>",
+    "<audio_1025>",
+    "<audio_1026>",
+    "<audio_1027>",
+    "<audio_1028>",
+    "<audio_1029>",
+    "<audio_1030>",
+    "<audio_1031>",
+    "<audio_1032>",
+    "<audio_1033>",
+    "<audio_1034>",
+    "<audio_1035>",
+    "<audio_1036>",
+    "<audio_1037>",
+    "<audio_1038>",
+    "<audio_1039>",
+    "<audio_1040>",
+    "<audio_1041>",
+    "<audio_1042>",
+    "<audio_1043>",
+    "<audio_1044>",
+    "<audio_1045>",
+    "<audio_1046>",
+    "<audio_1047>",
+    "<audio_1048>",
+    "<audio_1049>",
+    "<audio_1050>",
+    "<audio_1051>",
+    "<audio_1052>",
+    "<audio_1053>",
+    "<audio_1054>",
+    "<audio_1055>",
+    "<audio_1056>",
+    "<audio_1057>",
+    "<audio_1058>",
+    "<audio_1059>",
+    "<audio_1060>",
+    "<audio_1061>",
+    "<audio_1062>",
+    "<audio_1063>",
+    "<audio_1064>",
+    "<audio_1065>",
+    "<audio_1066>",
+    "<audio_1067>",
+    "<audio_1068>",
+    "<audio_1069>",
+    "<audio_1070>",
+    "<audio_1071>",
+    "<audio_1072>",
+    "<audio_1073>",
+    "<audio_1074>",
+    "<audio_1075>",
+    "<audio_1076>",
+    "<audio_1077>",
+    "<audio_1078>",
+    "<audio_1079>",
+    "<audio_1080>",
+    "<audio_1081>",
+    "<audio_1082>",
+    "<audio_1083>",
+    "<audio_1084>",
+    "<audio_1085>",
+    "<audio_1086>",
+    "<audio_1087>",
+    "<audio_1088>",
+    "<audio_1089>",
+    "<audio_1090>",
+    "<audio_1091>",
+    "<audio_1092>",
+    "<audio_1093>",
+    "<audio_1094>",
+    "<audio_1095>",
+    "<audio_1096>",
+    "<audio_1097>",
+    "<audio_1098>",
+    "<audio_1099>",
+    "<audio_1100>",
+    "<audio_1101>",
+    "<audio_1102>",
+    "<audio_1103>",
+    "<audio_1104>",
+    "<audio_1105>",
+    "<audio_1106>",
+    "<audio_1107>",
+    "<audio_1108>",
+    "<audio_1109>",
+    "<audio_1110>",
+    "<audio_1111>",
+    "<audio_1112>",
+    "<audio_1113>",
+    "<audio_1114>",
+    "<audio_1115>",
+    "<audio_1116>",
+    "<audio_1117>",
+    "<audio_1118>",
+    "<audio_1119>",
+    "<audio_1120>",
+    "<audio_1121>",
+    "<audio_1122>",
+    "<audio_1123>",
+    "<audio_1124>",
+    "<audio_1125>",
+    "<audio_1126>",
+    "<audio_1127>",
+    "<audio_1128>",
+    "<audio_1129>",
+    "<audio_1130>",
+    "<audio_1131>",
+    "<audio_1132>",
+    "<audio_1133>",
+    "<audio_1134>",
+    "<audio_1135>",
+    "<audio_1136>",
+    "<audio_1137>",
+    "<audio_1138>",
+    "<audio_1139>",
+    "<audio_1140>",
+    "<audio_1141>",
+    "<audio_1142>",
+    "<audio_1143>",
+    "<audio_1144>",
+    "<audio_1145>",
+    "<audio_1146>",
+    "<audio_1147>",
+    "<audio_1148>",
+    "<audio_1149>",
+    "<audio_1150>",
+    "<audio_1151>",
+    "<audio_1152>",
+    "<audio_1153>",
+    "<audio_1154>",
+    "<audio_1155>",
+    "<audio_1156>",
+    "<audio_1157>",
+    "<audio_1158>",
+    "<audio_1159>",
+    "<audio_1160>",
+    "<audio_1161>",
+    "<audio_1162>",
+    "<audio_1163>",
+    "<audio_1164>",
+    "<audio_1165>",
+    "<audio_1166>",
+    "<audio_1167>",
+    "<audio_1168>",
+    "<audio_1169>",
+    "<audio_1170>",
+    "<audio_1171>",
+    "<audio_1172>",
+    "<audio_1173>",
+    "<audio_1174>",
+    "<audio_1175>",
+    "<audio_1176>",
+    "<audio_1177>",
+    "<audio_1178>",
+    "<audio_1179>",
+    "<audio_1180>",
+    "<audio_1181>",
+    "<audio_1182>",
+    "<audio_1183>",
+    "<audio_1184>",
+    "<audio_1185>",
+    "<audio_1186>",
+    "<audio_1187>",
+    "<audio_1188>",
+    "<audio_1189>",
+    "<audio_1190>",
+    "<audio_1191>",
+    "<audio_1192>",
+    "<audio_1193>",
+    "<audio_1194>",
+    "<audio_1195>",
+    "<audio_1196>",
+    "<audio_1197>",
+    "<audio_1198>",
+    "<audio_1199>",
+    "<audio_1200>",
+    "<audio_1201>",
+    "<audio_1202>",
+    "<audio_1203>",
+    "<audio_1204>",
+    "<audio_1205>",
+    "<audio_1206>",
+    "<audio_1207>",
+    "<audio_1208>",
+    "<audio_1209>",
+    "<audio_1210>",
+    "<audio_1211>",
+    "<audio_1212>",
+    "<audio_1213>",
+    "<audio_1214>",
+    "<audio_1215>",
+    "<audio_1216>",
+    "<audio_1217>",
+    "<audio_1218>",
+    "<audio_1219>",
+    "<audio_1220>",
+    "<audio_1221>",
+    "<audio_1222>",
+    "<audio_1223>",
+    "<audio_1224>",
+    "<audio_1225>",
+    "<audio_1226>",
+    "<audio_1227>",
+    "<audio_1228>",
+    "<audio_1229>",
+    "<audio_1230>",
+    "<audio_1231>",
+    "<audio_1232>",
+    "<audio_1233>",
+    "<audio_1234>",
+    "<audio_1235>",
+    "<audio_1236>",
+    "<audio_1237>",
+    "<audio_1238>",
+    "<audio_1239>",
+    "<audio_1240>",
+    "<audio_1241>",
+    "<audio_1242>",
+    "<audio_1243>",
+    "<audio_1244>",
+    "<audio_1245>",
+    "<audio_1246>",
+    "<audio_1247>",
+    "<audio_1248>",
+    "<audio_1249>",
+    "<audio_1250>",
+    "<audio_1251>",
+    "<audio_1252>",
+    "<audio_1253>",
+    "<audio_1254>",
+    "<audio_1255>",
+    "<audio_1256>",
+    "<audio_1257>",
+    "<audio_1258>",
+    "<audio_1259>",
+    "<audio_1260>",
+    "<audio_1261>",
+    "<audio_1262>",
+    "<audio_1263>",
+    "<audio_1264>",
+    "<audio_1265>",
+    "<audio_1266>",
+    "<audio_1267>",
+    "<audio_1268>",
+    "<audio_1269>",
+    "<audio_1270>",
+    "<audio_1271>",
+    "<audio_1272>",
+    "<audio_1273>",
+    "<audio_1274>",
+    "<audio_1275>",
+    "<audio_1276>",
+    "<audio_1277>",
+    "<audio_1278>",
+    "<audio_1279>",
+    "<audio_1280>",
+    "<audio_1281>",
+    "<audio_1282>",
+    "<audio_1283>",
+    "<audio_1284>",
+    "<audio_1285>",
+    "<audio_1286>",
+    "<audio_1287>",
+    "<audio_1288>",
+    "<audio_1289>",
+    "<audio_1290>",
+    "<audio_1291>",
+    "<audio_1292>",
+    "<audio_1293>",
+    "<audio_1294>",
+    "<audio_1295>",
+    "<audio_1296>",
+    "<audio_1297>",
+    "<audio_1298>",
+    "<audio_1299>",
+    "<audio_1300>",
+    "<audio_1301>",
+    "<audio_1302>",
+    "<audio_1303>",
+    "<audio_1304>",
+    "<audio_1305>",
+    "<audio_1306>",
+    "<audio_1307>",
+    "<audio_1308>",
+    "<audio_1309>",
+    "<audio_1310>",
+    "<audio_1311>",
+    "<audio_1312>",
+    "<audio_1313>",
+    "<audio_1314>",
+    "<audio_1315>",
+    "<audio_1316>",
+    "<audio_1317>",
+    "<audio_1318>",
+    "<audio_1319>",
+    "<audio_1320>",
+    "<audio_1321>",
+    "<audio_1322>",
+    "<audio_1323>",
+    "<audio_1324>",
+    "<audio_1325>",
+    "<audio_1326>",
+    "<audio_1327>",
+    "<audio_1328>",
+    "<audio_1329>",
+    "<audio_1330>",
+    "<audio_1331>",
+    "<audio_1332>",
+    "<audio_1333>",
+    "<audio_1334>",
+    "<audio_1335>",
+    "<audio_1336>",
+    "<audio_1337>",
+    "<audio_1338>",
+    "<audio_1339>",
+    "<audio_1340>",
+    "<audio_1341>",
+    "<audio_1342>",
+    "<audio_1343>",
+    "<audio_1344>",
+    "<audio_1345>",
+    "<audio_1346>",
+    "<audio_1347>",
+    "<audio_1348>",
+    "<audio_1349>",
+    "<audio_1350>",
+    "<audio_1351>",
+    "<audio_1352>",
+    "<audio_1353>",
+    "<audio_1354>",
+    "<audio_1355>",
+    "<audio_1356>",
+    "<audio_1357>",
+    "<audio_1358>",
+    "<audio_1359>",
+    "<audio_1360>",
+    "<audio_1361>",
+    "<audio_1362>",
+    "<audio_1363>",
+    "<audio_1364>",
+    "<audio_1365>",
+    "<audio_1366>",
+    "<audio_1367>",
+    "<audio_1368>",
+    "<audio_1369>",
+    "<audio_1370>",
+    "<audio_1371>",
+    "<audio_1372>",
+    "<audio_1373>",
+    "<audio_1374>",
+    "<audio_1375>",
+    "<audio_1376>",
+    "<audio_1377>",
+    "<audio_1378>",
+    "<audio_1379>",
+    "<audio_1380>",
+    "<audio_1381>",
+    "<audio_1382>",
+    "<audio_1383>",
+    "<audio_1384>",
+    "<audio_1385>",
+    "<audio_1386>",
+    "<audio_1387>",
+    "<audio_1388>",
+    "<audio_1389>",
+    "<audio_1390>",
+    "<audio_1391>",
+    "<audio_1392>",
+    "<audio_1393>",
+    "<audio_1394>",
+    "<audio_1395>",
+    "<audio_1396>",
+    "<audio_1397>",
+    "<audio_1398>",
+    "<audio_1399>",
+    "<audio_1400>",
+    "<audio_1401>",
+    "<audio_1402>",
+    "<audio_1403>",
+    "<audio_1404>",
+    "<audio_1405>",
+    "<audio_1406>",
+    "<audio_1407>",
+    "<audio_1408>",
+    "<audio_1409>",
+    "<audio_1410>",
+    "<audio_1411>",
+    "<audio_1412>",
+    "<audio_1413>",
+    "<audio_1414>",
+    "<audio_1415>",
+    "<audio_1416>",
+    "<audio_1417>",
+    "<audio_1418>",
+    "<audio_1419>",
+    "<audio_1420>",
+    "<audio_1421>",
+    "<audio_1422>",
+    "<audio_1423>",
+    "<audio_1424>",
+    "<audio_1425>",
+    "<audio_1426>",
+    "<audio_1427>",
+    "<audio_1428>",
+    "<audio_1429>",
+    "<audio_1430>",
+    "<audio_1431>",
+    "<audio_1432>",
+    "<audio_1433>",
+    "<audio_1434>",
+    "<audio_1435>",
+    "<audio_1436>",
+    "<audio_1437>",
+    "<audio_1438>",
+    "<audio_1439>",
+    "<audio_1440>",
+    "<audio_1441>",
+    "<audio_1442>",
+    "<audio_1443>",
+    "<audio_1444>",
+    "<audio_1445>",
+    "<audio_1446>",
+    "<audio_1447>",
+    "<audio_1448>",
+    "<audio_1449>",
+    "<audio_1450>",
+    "<audio_1451>",
+    "<audio_1452>",
+    "<audio_1453>",
+    "<audio_1454>",
+    "<audio_1455>",
+    "<audio_1456>",
+    "<audio_1457>",
+    "<audio_1458>",
+    "<audio_1459>",
+    "<audio_1460>",
+    "<audio_1461>",
+    "<audio_1462>",
+    "<audio_1463>",
+    "<audio_1464>",
+    "<audio_1465>",
+    "<audio_1466>",
+    "<audio_1467>",
+    "<audio_1468>",
+    "<audio_1469>",
+    "<audio_1470>",
+    "<audio_1471>",
+    "<audio_1472>",
+    "<audio_1473>",
+    "<audio_1474>",
+    "<audio_1475>",
+    "<audio_1476>",
+    "<audio_1477>",
+    "<audio_1478>",
+    "<audio_1479>",
+    "<audio_1480>",
+    "<audio_1481>",
+    "<audio_1482>",
+    "<audio_1483>",
+    "<audio_1484>",
+    "<audio_1485>",
+    "<audio_1486>",
+    "<audio_1487>",
+    "<audio_1488>",
+    "<audio_1489>",
+    "<audio_1490>",
+    "<audio_1491>",
+    "<audio_1492>",
+    "<audio_1493>",
+    "<audio_1494>",
+    "<audio_1495>",
+    "<audio_1496>",
+    "<audio_1497>",
+    "<audio_1498>",
+    "<audio_1499>",
+    "<audio_1500>",
+    "<audio_1501>",
+    "<audio_1502>",
+    "<audio_1503>",
+    "<audio_1504>",
+    "<audio_1505>",
+    "<audio_1506>",
+    "<audio_1507>",
+    "<audio_1508>",
+    "<audio_1509>",
+    "<audio_1510>",
+    "<audio_1511>",
+    "<audio_1512>",
+    "<audio_1513>",
+    "<audio_1514>",
+    "<audio_1515>",
+    "<audio_1516>",
+    "<audio_1517>",
+    "<audio_1518>",
+    "<audio_1519>",
+    "<audio_1520>",
+    "<audio_1521>",
+    "<audio_1522>",
+    "<audio_1523>",
+    "<audio_1524>",
+    "<audio_1525>",
+    "<audio_1526>",
+    "<audio_1527>",
+    "<audio_1528>",
+    "<audio_1529>",
+    "<audio_1530>",
+    "<audio_1531>",
+    "<audio_1532>",
+    "<audio_1533>",
+    "<audio_1534>",
+    "<audio_1535>",
+    "<audio_1536>",
+    "<audio_1537>",
+    "<audio_1538>",
+    "<audio_1539>",
+    "<audio_1540>",
+    "<audio_1541>",
+    "<audio_1542>",
+    "<audio_1543>",
+    "<audio_1544>",
+    "<audio_1545>",
+    "<audio_1546>",
+    "<audio_1547>",
+    "<audio_1548>",
+    "<audio_1549>",
+    "<audio_1550>",
+    "<audio_1551>",
+    "<audio_1552>",
+    "<audio_1553>",
+    "<audio_1554>",
+    "<audio_1555>",
+    "<audio_1556>",
+    "<audio_1557>",
+    "<audio_1558>",
+    "<audio_1559>",
+    "<audio_1560>",
+    "<audio_1561>",
+    "<audio_1562>",
+    "<audio_1563>",
+    "<audio_1564>",
+    "<audio_1565>",
+    "<audio_1566>",
+    "<audio_1567>",
+    "<audio_1568>",
+    "<audio_1569>",
+    "<audio_1570>",
+    "<audio_1571>",
+    "<audio_1572>",
+    "<audio_1573>",
+    "<audio_1574>",
+    "<audio_1575>",
+    "<audio_1576>",
+    "<audio_1577>",
+    "<audio_1578>",
+    "<audio_1579>",
+    "<audio_1580>",
+    "<audio_1581>",
+    "<audio_1582>",
+    "<audio_1583>",
+    "<audio_1584>",
+    "<audio_1585>",
+    "<audio_1586>",
+    "<audio_1587>",
+    "<audio_1588>",
+    "<audio_1589>",
+    "<audio_1590>",
+    "<audio_1591>",
+    "<audio_1592>",
+    "<audio_1593>",
+    "<audio_1594>",
+    "<audio_1595>",
+    "<audio_1596>",
+    "<audio_1597>",
+    "<audio_1598>",
+    "<audio_1599>",
+    "<audio_1600>",
+    "<audio_1601>",
+    "<audio_1602>",
+    "<audio_1603>",
+    "<audio_1604>",
+    "<audio_1605>",
+    "<audio_1606>",
+    "<audio_1607>",
+    "<audio_1608>",
+    "<audio_1609>",
+    "<audio_1610>",
+    "<audio_1611>",
+    "<audio_1612>",
+    "<audio_1613>",
+    "<audio_1614>",
+    "<audio_1615>",
+    "<audio_1616>",
+    "<audio_1617>",
+    "<audio_1618>",
+    "<audio_1619>",
+    "<audio_1620>",
+    "<audio_1621>",
+    "<audio_1622>",
+    "<audio_1623>",
+    "<audio_1624>",
+    "<audio_1625>",
+    "<audio_1626>",
+    "<audio_1627>",
+    "<audio_1628>",
+    "<audio_1629>",
+    "<audio_1630>",
+    "<audio_1631>",
+    "<audio_1632>",
+    "<audio_1633>",
+    "<audio_1634>",
+    "<audio_1635>",
+    "<audio_1636>",
+    "<audio_1637>",
+    "<audio_1638>",
+    "<audio_1639>",
+    "<audio_1640>",
+    "<audio_1641>",
+    "<audio_1642>",
+    "<audio_1643>",
+    "<audio_1644>",
+    "<audio_1645>",
+    "<audio_1646>",
+    "<audio_1647>",
+    "<audio_1648>",
+    "<audio_1649>",
+    "<audio_1650>",
+    "<audio_1651>",
+    "<audio_1652>",
+    "<audio_1653>",
+    "<audio_1654>",
+    "<audio_1655>",
+    "<audio_1656>",
+    "<audio_1657>",
+    "<audio_1658>",
+    "<audio_1659>",
+    "<audio_1660>",
+    "<audio_1661>",
+    "<audio_1662>",
+    "<audio_1663>",
+    "<audio_1664>",
+    "<audio_1665>",
+    "<audio_1666>",
+    "<audio_1667>",
+    "<audio_1668>",
+    "<audio_1669>",
+    "<audio_1670>",
+    "<audio_1671>",
+    "<audio_1672>",
+    "<audio_1673>",
+    "<audio_1674>",
+    "<audio_1675>",
+    "<audio_1676>",
+    "<audio_1677>",
+    "<audio_1678>",
+    "<audio_1679>",
+    "<audio_1680>",
+    "<audio_1681>",
+    "<audio_1682>",
+    "<audio_1683>",
+    "<audio_1684>",
+    "<audio_1685>",
+    "<audio_1686>",
+    "<audio_1687>",
+    "<audio_1688>",
+    "<audio_1689>",
+    "<audio_1690>",
+    "<audio_1691>",
+    "<audio_1692>",
+    "<audio_1693>",
+    "<audio_1694>",
+    "<audio_1695>",
+    "<audio_1696>",
+    "<audio_1697>",
+    "<audio_1698>",
+    "<audio_1699>",
+    "<audio_1700>",
+    "<audio_1701>",
+    "<audio_1702>",
+    "<audio_1703>",
+    "<audio_1704>",
+    "<audio_1705>",
+    "<audio_1706>",
+    "<audio_1707>",
+    "<audio_1708>",
+    "<audio_1709>",
+    "<audio_1710>",
+    "<audio_1711>",
+    "<audio_1712>",
+    "<audio_1713>",
+    "<audio_1714>",
+    "<audio_1715>",
+    "<audio_1716>",
+    "<audio_1717>",
+    "<audio_1718>",
+    "<audio_1719>",
+    "<audio_1720>",
+    "<audio_1721>",
+    "<audio_1722>",
+    "<audio_1723>",
+    "<audio_1724>",
+    "<audio_1725>",
+    "<audio_1726>",
+    "<audio_1727>",
+    "<audio_1728>",
+    "<audio_1729>",
+    "<audio_1730>",
+    "<audio_1731>",
+    "<audio_1732>",
+    "<audio_1733>",
+    "<audio_1734>",
+    "<audio_1735>",
+    "<audio_1736>",
+    "<audio_1737>",
+    "<audio_1738>",
+    "<audio_1739>",
+    "<audio_1740>",
+    "<audio_1741>",
+    "<audio_1742>",
+    "<audio_1743>",
+    "<audio_1744>",
+    "<audio_1745>",
+    "<audio_1746>",
+    "<audio_1747>",
+    "<audio_1748>",
+    "<audio_1749>",
+    "<audio_1750>",
+    "<audio_1751>",
+    "<audio_1752>",
+    "<audio_1753>",
+    "<audio_1754>",
+    "<audio_1755>",
+    "<audio_1756>",
+    "<audio_1757>",
+    "<audio_1758>",
+    "<audio_1759>",
+    "<audio_1760>",
+    "<audio_1761>",
+    "<audio_1762>",
+    "<audio_1763>",
+    "<audio_1764>",
+    "<audio_1765>",
+    "<audio_1766>",
+    "<audio_1767>",
+    "<audio_1768>",
+    "<audio_1769>",
+    "<audio_1770>",
+    "<audio_1771>",
+    "<audio_1772>",
+    "<audio_1773>",
+    "<audio_1774>",
+    "<audio_1775>",
+    "<audio_1776>",
+    "<audio_1777>",
+    "<audio_1778>",
+    "<audio_1779>",
+    "<audio_1780>",
+    "<audio_1781>",
+    "<audio_1782>",
+    "<audio_1783>",
+    "<audio_1784>",
+    "<audio_1785>",
+    "<audio_1786>",
+    "<audio_1787>",
+    "<audio_1788>",
+    "<audio_1789>",
+    "<audio_1790>",
+    "<audio_1791>",
+    "<audio_1792>",
+    "<audio_1793>",
+    "<audio_1794>",
+    "<audio_1795>",
+    "<audio_1796>",
+    "<audio_1797>",
+    "<audio_1798>",
+    "<audio_1799>",
+    "<audio_1800>",
+    "<audio_1801>",
+    "<audio_1802>",
+    "<audio_1803>",
+    "<audio_1804>",
+    "<audio_1805>",
+    "<audio_1806>",
+    "<audio_1807>",
+    "<audio_1808>",
+    "<audio_1809>",
+    "<audio_1810>",
+    "<audio_1811>",
+    "<audio_1812>",
+    "<audio_1813>",
+    "<audio_1814>",
+    "<audio_1815>",
+    "<audio_1816>",
+    "<audio_1817>",
+    "<audio_1818>",
+    "<audio_1819>",
+    "<audio_1820>",
+    "<audio_1821>",
+    "<audio_1822>",
+    "<audio_1823>",
+    "<audio_1824>",
+    "<audio_1825>",
+    "<audio_1826>",
+    "<audio_1827>",
+    "<audio_1828>",
+    "<audio_1829>",
+    "<audio_1830>",
+    "<audio_1831>",
+    "<audio_1832>",
+    "<audio_1833>",
+    "<audio_1834>",
+    "<audio_1835>",
+    "<audio_1836>",
+    "<audio_1837>",
+    "<audio_1838>",
+    "<audio_1839>",
+    "<audio_1840>",
+    "<audio_1841>",
+    "<audio_1842>",
+    "<audio_1843>",
+    "<audio_1844>",
+    "<audio_1845>",
+    "<audio_1846>",
+    "<audio_1847>",
+    "<audio_1848>",
+    "<audio_1849>",
+    "<audio_1850>",
+    "<audio_1851>",
+    "<audio_1852>",
+    "<audio_1853>",
+    "<audio_1854>",
+    "<audio_1855>",
+    "<audio_1856>",
+    "<audio_1857>",
+    "<audio_1858>",
+    "<audio_1859>",
+    "<audio_1860>",
+    "<audio_1861>",
+    "<audio_1862>",
+    "<audio_1863>",
+    "<audio_1864>",
+    "<audio_1865>",
+    "<audio_1866>",
+    "<audio_1867>",
+    "<audio_1868>",
+    "<audio_1869>",
+    "<audio_1870>",
+    "<audio_1871>",
+    "<audio_1872>",
+    "<audio_1873>",
+    "<audio_1874>",
+    "<audio_1875>",
+    "<audio_1876>",
+    "<audio_1877>",
+    "<audio_1878>",
+    "<audio_1879>",
+    "<audio_1880>",
+    "<audio_1881>",
+    "<audio_1882>",
+    "<audio_1883>",
+    "<audio_1884>",
+    "<audio_1885>",
+    "<audio_1886>",
+    "<audio_1887>",
+    "<audio_1888>",
+    "<audio_1889>",
+    "<audio_1890>",
+    "<audio_1891>",
+    "<audio_1892>",
+    "<audio_1893>",
+    "<audio_1894>",
+    "<audio_1895>",
+    "<audio_1896>",
+    "<audio_1897>",
+    "<audio_1898>",
+    "<audio_1899>",
+    "<audio_1900>",
+    "<audio_1901>",
+    "<audio_1902>",
+    "<audio_1903>",
+    "<audio_1904>",
+    "<audio_1905>",
+    "<audio_1906>",
+    "<audio_1907>",
+    "<audio_1908>",
+    "<audio_1909>",
+    "<audio_1910>",
+    "<audio_1911>",
+    "<audio_1912>",
+    "<audio_1913>",
+    "<audio_1914>",
+    "<audio_1915>",
+    "<audio_1916>",
+    "<audio_1917>",
+    "<audio_1918>",
+    "<audio_1919>",
+    "<audio_1920>",
+    "<audio_1921>",
+    "<audio_1922>",
+    "<audio_1923>",
+    "<audio_1924>",
+    "<audio_1925>",
+    "<audio_1926>",
+    "<audio_1927>",
+    "<audio_1928>",
+    "<audio_1929>",
+    "<audio_1930>",
+    "<audio_1931>",
+    "<audio_1932>",
+    "<audio_1933>",
+    "<audio_1934>",
+    "<audio_1935>",
+    "<audio_1936>",
+    "<audio_1937>",
+    "<audio_1938>",
+    "<audio_1939>",
+    "<audio_1940>",
+    "<audio_1941>",
+    "<audio_1942>",
+    "<audio_1943>",
+    "<audio_1944>",
+    "<audio_1945>",
+    "<audio_1946>",
+    "<audio_1947>",
+    "<audio_1948>",
+    "<audio_1949>",
+    "<audio_1950>",
+    "<audio_1951>",
+    "<audio_1952>",
+    "<audio_1953>",
+    "<audio_1954>",
+    "<audio_1955>",
+    "<audio_1956>",
+    "<audio_1957>",
+    "<audio_1958>",
+    "<audio_1959>",
+    "<audio_1960>",
+    "<audio_1961>",
+    "<audio_1962>",
+    "<audio_1963>",
+    "<audio_1964>",
+    "<audio_1965>",
+    "<audio_1966>",
+    "<audio_1967>",
+    "<audio_1968>",
+    "<audio_1969>",
+    "<audio_1970>",
+    "<audio_1971>",
+    "<audio_1972>",
+    "<audio_1973>",
+    "<audio_1974>",
+    "<audio_1975>",
+    "<audio_1976>",
+    "<audio_1977>",
+    "<audio_1978>",
+    "<audio_1979>",
+    "<audio_1980>",
+    "<audio_1981>",
+    "<audio_1982>",
+    "<audio_1983>",
+    "<audio_1984>",
+    "<audio_1985>",
+    "<audio_1986>",
+    "<audio_1987>",
+    "<audio_1988>",
+    "<audio_1989>",
+    "<audio_1990>",
+    "<audio_1991>",
+    "<audio_1992>",
+    "<audio_1993>",
+    "<audio_1994>",
+    "<audio_1995>",
+    "<audio_1996>",
+    "<audio_1997>",
+    "<audio_1998>",
+    "<audio_1999>",
+    "<audio_2000>",
+    "<audio_2001>",
+    "<audio_2002>",
+    "<audio_2003>",
+    "<audio_2004>",
+    "<audio_2005>",
+    "<audio_2006>",
+    "<audio_2007>",
+    "<audio_2008>",
+    "<audio_2009>",
+    "<audio_2010>",
+    "<audio_2011>",
+    "<audio_2012>",
+    "<audio_2013>",
+    "<audio_2014>",
+    "<audio_2015>",
+    "<audio_2016>",
+    "<audio_2017>",
+    "<audio_2018>",
+    "<audio_2019>",
+    "<audio_2020>",
+    "<audio_2021>",
+    "<audio_2022>",
+    "<audio_2023>",
+    "<audio_2024>",
+    "<audio_2025>",
+    "<audio_2026>",
+    "<audio_2027>",
+    "<audio_2028>",
+    "<audio_2029>",
+    "<audio_2030>",
+    "<audio_2031>",
+    "<audio_2032>",
+    "<audio_2033>",
+    "<audio_2034>",
+    "<audio_2035>",
+    "<audio_2036>",
+    "<audio_2037>",
+    "<audio_2038>",
+    "<audio_2039>",
+    "<audio_2040>",
+    "<audio_2041>",
+    "<audio_2042>",
+    "<audio_2043>",
+    "<audio_2044>",
+    "<audio_2045>",
+    "<audio_2046>",
+    "<audio_2047>",
+    "<spk1>",
+    "<spk2>",
+    "<spk3>",
+    "<spk4>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

token_manifest.json ADDED Viewed

	@@ -0,0 +1,14 @@

+{
+  "soa_token_id": 152069,
+  "eoa_token_id": 152070,
+  "codebook_offset": 152072,
+  "codebook_size": 2048,
+  "spk_token_ids": [
+    154120,
+    154121,
+    154122,
+    154123
+  ],
+  "eos_token_id": 151643,
+  "max_audio_codes": 1000
+}

tokenizer.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:aca5608200fb11ad1546df5f48416de24c6920e8d8515b4f5e35024516fd3fc4
+size 11888678

tokenizer_config.json ADDED Viewed

The diff for this file is too large to render. See raw diff

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff