diff --git a/Ins/added_tokens.json b/Ins/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..d98719c951a52f071b6df8945c2261f26413dff1 --- /dev/null +++ b/Ins/added_tokens.json @@ -0,0 +1,1026 @@ +{ + "": 32000, + "": 32100, + "": 32101, + "": 32102, + "": 32103, + "": 32104, + "": 32105, + "": 32106, + "": 32107, + "": 32108, + "": 32109, + "": 32010, + "": 32110, + "": 32111, + "": 32112, + "": 32113, + "": 32114, + "": 32115, + "": 32116, + "": 32117, + "": 32118, + "": 32119, + "": 32011, + "": 32120, + "": 32121, + "": 32122, + "": 32123, + "": 32124, + "": 32125, + "": 32126, + "": 32127, + "": 32128, + "": 32129, + "": 32012, + "": 32130, + "": 32131, + "": 32132, + "": 32133, + "": 32134, + "": 32135, + "": 32136, + "": 32137, + "": 32138, + "": 32139, + "": 32013, + "": 32140, + "": 32141, + "": 32142, + "": 32143, + "": 32144, + "": 32145, + "": 32146, + "": 32147, + "": 32148, + "": 32149, + "": 32014, + "": 32150, + "": 32151, + "": 32152, + "": 32153, + "": 32154, + "": 32155, + "": 32156, + "": 32157, + "": 32158, + "": 32159, + "": 32015, + "": 32160, + "": 32161, + "": 32162, + "": 32163, + "": 32164, + "": 32165, + "": 32166, + "": 32167, + "": 32168, + "": 32169, + "": 32016, + "": 32170, + "": 32171, + "": 32172, + "": 32173, + "": 32174, + "": 32175, + "": 32176, + "": 32177, + "": 32178, + "": 32179, + "": 32017, + "": 32180, + "": 32181, + "": 32182, + "": 32183, + "": 32184, + "": 32185, + "": 32186, + "": 32187, + "": 32188, + "": 32189, + "": 32018, + "": 32190, + "": 32191, + "": 32192, + "": 32193, + "": 32194, + "": 32195, + "": 32196, + "": 32197, + "": 32198, + "": 32199, + "": 32019, + "": 32001, + "": 32200, + "": 32201, + "": 32202, + "": 32203, + "": 32204, + "": 32205, + "": 32206, + "": 32207, + "": 32208, + "": 32209, + "": 32020, + "": 32210, + "": 32211, + "": 32212, + "": 32213, + "": 32214, + "": 32215, + "": 32216, + "": 32217, + "": 32218, + "": 32219, + "": 32021, + "": 32220, + "": 32221, + "": 32222, + "": 32223, + "": 32224, + "": 32225, + "": 32226, + "": 32227, + "": 32228, + "": 32229, + "": 32022, + "": 32230, + "": 32231, + "": 32232, + "": 32233, + "": 32234, + "": 32235, + "": 32236, + "": 32237, + "": 32238, + "": 32239, + "": 32023, + "": 32240, + "": 32241, + "": 32242, + "": 32243, + "": 32244, + "": 32245, + "": 32246, + "": 32247, + "": 32248, + "": 32249, + "": 32024, + "": 32250, + "": 32251, + "": 32252, + "": 32253, + "": 32254, + "": 32255, + "": 32025, + "": 32026, + "": 32027, + "": 32028, + "": 32029, + "": 32002, + "": 32030, + "": 32031, + "": 32032, + "": 32033, + "": 32034, + "": 32035, + "": 32036, + "": 32037, + "": 32038, + "": 32039, + "": 32003, + "": 32040, + "": 32041, + "": 32042, + "": 32043, + "": 32044, + "": 32045, + "": 32046, + "": 32047, + "": 32048, + "": 32049, + "": 32004, + "": 32050, + "": 32051, + "": 32052, + "": 32053, + "": 32054, + "": 32055, + "": 32056, + "": 32057, + "": 32058, + "": 32059, + "": 32005, + "": 32060, + "": 32061, + "": 32062, + "": 32063, + "": 32064, + "": 32065, + "": 32066, + "": 32067, + "": 32068, + "": 32069, + "": 32006, + "": 32070, + "": 32071, + "": 32072, + "": 32073, + "": 32074, + "": 32075, + "": 32076, + "": 32077, + "": 32078, + "": 32079, + "": 32007, + "": 32080, + "": 32081, + "": 32082, + "": 32083, + "": 32084, + "": 32085, + "": 32086, + "": 32087, + "": 32088, + "": 32089, + "": 32008, + "": 32090, + "": 32091, + "": 32092, + "": 32093, + "": 32094, + "": 32095, + "": 32096, + "": 32097, + "": 32098, + "": 32099, + "": 32009, + "": 32256, + "": 32356, + "": 32357, + "": 32358, + "": 32359, + "": 32360, + "": 32361, + "": 32362, + "": 32363, + "": 32364, + "": 32365, + "": 32266, + "": 32366, + "": 32367, + "": 32368, + "": 32369, + "": 32370, + "": 32371, + "": 32372, + "": 32373, + "": 32374, + "": 32375, + "": 32267, + "": 32376, + "": 32377, + "": 32378, + "": 32379, + "": 32380, + "": 32381, + "": 32382, + "": 32383, + "": 32384, + "": 32385, + "": 32268, + "": 32386, + "": 32387, + "": 32388, + "": 32389, + "": 32390, + "": 32391, + "": 32392, + "": 32393, + "": 32394, + "": 32395, + "": 32269, + "": 32396, + "": 32397, + "": 32398, + "": 32399, + "": 32400, + "": 32401, + "": 32402, + "": 32403, + "": 32404, + "": 32405, + "": 32270, + "": 32406, + "": 32407, + "": 32408, + "": 32409, + "": 32410, + "": 32411, + "": 32412, + "": 32413, + "": 32414, + "": 32415, + "": 32271, + "": 32416, + "": 32417, + "": 32418, + "": 32419, + "": 32420, + "": 32421, + "": 32422, + "": 32423, + "": 32424, + "": 32425, + "": 32272, + "": 32426, + "": 32427, + "": 32428, + "": 32429, + "": 32430, + "": 32431, + "": 32432, + "": 32433, + "": 32434, + "": 32435, + "": 32273, + "": 32436, + "": 32437, + "": 32438, + "": 32439, + "": 32440, + "": 32441, + "": 32442, + "": 32443, + "": 32444, + "": 32445, + "": 32274, + "": 32446, + "": 32447, + "": 32448, + "": 32449, + "": 32450, + "": 32451, + "": 32452, + "": 32453, + "": 32454, + "": 32455, + "": 32275, + "": 32257, + "": 32456, + "": 32457, + "": 32458, + "": 32459, + "": 32460, + "": 32461, + "": 32462, + "": 32463, + "": 32464, + "": 32465, + "": 32276, + "": 32466, + "": 32467, + "": 32468, + "": 32469, + "": 32470, + "": 32471, + "": 32472, + "": 32473, + "": 32474, + "": 32475, + "": 32277, + "": 32476, + "": 32477, + "": 32478, + "": 32479, + "": 32480, + "": 32481, + "": 32482, + "": 32483, + "": 32484, + "": 32485, + "": 32278, + "": 32486, + "": 32487, + "": 32488, + "": 32489, + "": 32490, + "": 32491, + "": 32492, + "": 32493, + "": 32494, + "": 32495, + "": 32279, + "": 32496, + "": 32497, + "": 32498, + "": 32499, + "": 32500, + "": 32501, + "": 32502, + "": 32503, + "": 32504, + "": 32505, + "": 32280, + "": 32506, + "": 32507, + "": 32508, + "": 32509, + "": 32510, + "": 32511, + "": 32281, + "": 32282, + "": 32283, + "": 32284, + "": 32285, + "": 32258, + "": 32286, + "": 32287, + "": 32288, + "": 32289, + "": 32290, + "": 32291, + "": 32292, + "": 32293, + "": 32294, + "": 32295, + "": 32259, + "": 32296, + "": 32297, + "": 32298, + "": 32299, + "": 32300, + "": 32301, + "": 32302, + "": 32303, + "": 32304, + "": 32305, + "": 32260, + "": 32306, + "": 32307, + "": 32308, + "": 32309, + "": 32310, + "": 32311, + "": 32312, + "": 32313, + "": 32314, + "": 32315, + "": 32261, + "": 32316, + "": 32317, + "": 32318, + "": 32319, + "": 32320, + "": 32321, + "": 32322, + "": 32323, + "": 32324, + "": 32325, + "": 32262, + "": 32326, + "": 32327, + "": 32328, + "": 32329, + "": 32330, + "": 32331, + "": 32332, + "": 32333, + "": 32334, + "": 32335, + "": 32263, + "": 32336, + "": 32337, + "": 32338, + "": 32339, + "": 32340, + "": 32341, + "": 32342, + "": 32343, + "": 32344, + "": 32345, + "": 32264, + "": 32346, + "": 32347, + "": 32348, + "": 32349, + "": 32350, + "": 32351, + "": 32352, + "": 32353, + "": 32354, + "": 32355, + "": 32265, + "": 32512, + "": 32612, + "": 32613, + "": 32614, + "": 32615, + "": 32616, + "": 32617, + "": 32618, + "": 32619, + "": 32620, + "": 32621, + "": 32522, + "": 32622, + "": 32623, + "": 32624, + "": 32625, + "": 32626, + "": 32627, + "": 32628, + "": 32629, + "": 32630, + "": 32631, + "": 32523, + "": 32632, + "": 32633, + "": 32634, + "": 32635, + "": 32636, + "": 32637, + "": 32638, + "": 32639, + "": 32640, + "": 32641, + "": 32524, + "": 32642, + "": 32643, + "": 32644, + "": 32645, + "": 32646, + "": 32647, + "": 32648, + "": 32649, + "": 32650, + "": 32651, + "": 32525, + "": 32652, + "": 32653, + "": 32654, + "": 32655, + "": 32656, + "": 32657, + "": 32658, + "": 32659, + "": 32660, + "": 32661, + "": 32526, + "": 32662, + "": 32663, + "": 32664, + "": 32665, + "": 32666, + "": 32667, + "": 32668, + "": 32669, + "": 32670, + "": 32671, + "": 32527, + "": 32672, + "": 32673, + "": 32674, + "": 32675, + "": 32676, + "": 32677, + "": 32678, + "": 32679, + "": 32680, + "": 32681, + "": 32528, + "": 32682, + "": 32683, + "": 32684, + "": 32685, + "": 32686, + "": 32687, + "": 32688, + "": 32689, + "": 32690, + "": 32691, + "": 32529, + "": 32692, + "": 32693, + "": 32694, + "": 32695, + "": 32696, + "": 32697, + "": 32698, + "": 32699, + "": 32700, + "": 32701, + "": 32530, + "": 32702, + "": 32703, + "": 32704, + "": 32705, + "": 32706, + "": 32707, + "": 32708, + "": 32709, + "": 32710, + "": 32711, + "": 32531, + "": 32513, + "": 32712, + "": 32713, + "": 32714, + "": 32715, + "": 32716, + "": 32717, + "": 32718, + "": 32719, + "": 32720, + "": 32721, + "": 32532, + "": 32722, + "": 32723, + "": 32724, + "": 32725, + "": 32726, + "": 32727, + "": 32728, + "": 32729, + "": 32730, + "": 32731, + "": 32533, + "": 32732, + "": 32733, + "": 32734, + "": 32735, + "": 32736, + "": 32737, + "": 32738, + "": 32739, + "": 32740, + "": 32741, + "": 32534, + "": 32742, + "": 32743, + "": 32744, + "": 32745, + "": 32746, + "": 32747, + "": 32748, + "": 32749, + "": 32750, + "": 32751, + "": 32535, + "": 32752, + "": 32753, + "": 32754, + "": 32755, + "": 32756, + "": 32757, + "": 32758, + "": 32759, + "": 32760, + "": 32761, + "": 32536, + "": 32762, + "": 32763, + "": 32764, + "": 32765, + "": 32766, + "": 32767, + "": 32537, + "": 32538, + "": 32539, + "": 32540, + "": 32541, + "": 32514, + "": 32542, + "": 32543, + "": 32544, + "": 32545, + "": 32546, + "": 32547, + "": 32548, + "": 32549, + "": 32550, + "": 32551, + "": 32515, + "": 32552, + "": 32553, + "": 32554, + "": 32555, + "": 32556, + "": 32557, + "": 32558, + "": 32559, + "": 32560, + "": 32561, + "": 32516, + "": 32562, + "": 32563, + "": 32564, + "": 32565, + "": 32566, + "": 32567, + "": 32568, + "": 32569, + "": 32570, + "": 32571, + "": 32517, + "": 32572, + "": 32573, + "": 32574, + "": 32575, + "": 32576, + "": 32577, + "": 32578, + "": 32579, + "": 32580, + "": 32581, + "": 32518, + "": 32582, + "": 32583, + "": 32584, + "": 32585, + "": 32586, + "": 32587, + "": 32588, + "": 32589, + "": 32590, + "": 32591, + "": 32519, + "": 32592, + "": 32593, + "": 32594, + "": 32595, + "": 32596, + "": 32597, + "": 32598, + "": 32599, + "": 32600, + "": 32601, + "": 32520, + "": 32602, + "": 32603, + "": 32604, + "": 32605, + "": 32606, + "": 32607, + "": 32608, + "": 32609, + "": 32610, + "": 32611, + "": 32521, + "": 32768, + "": 32868, + "": 32869, + "": 32870, + "": 32871, + "": 32872, + "": 32873, + "": 32874, + "": 32875, + "": 32876, + "": 32877, + "": 32778, + "": 32878, + "": 32879, + "": 32880, + "": 32881, + "": 32882, + "": 32883, + "": 32884, + "": 32885, + "": 32886, + "": 32887, + "": 32779, + "": 32888, + "": 32889, + "": 32890, + "": 32891, + "": 32892, + "": 32893, + "": 32894, + "": 32895, + "": 32896, + "": 32897, + "": 32780, + "": 32898, + "": 32899, + "": 32900, + "": 32901, + "": 32902, + "": 32903, + "": 32904, + "": 32905, + "": 32906, + "": 32907, + "": 32781, + "": 32908, + "": 32909, + "": 32910, + "": 32911, + "": 32912, + "": 32913, + "": 32914, + "": 32915, + "": 32916, + "": 32917, + "": 32782, + "": 32918, + "": 32919, + "": 32920, + "": 32921, + "": 32922, + "": 32923, + "": 32924, + "": 32925, + "": 32926, + "": 32927, + "": 32783, + "": 32928, + "": 32929, + "": 32930, + "": 32931, + "": 32932, + "": 32933, + "": 32934, + "": 32935, + "": 32936, + "": 32937, + "": 32784, + "": 32938, + "": 32939, + "": 32940, + "": 32941, + "": 32942, + "": 32943, + "": 32944, + "": 32945, + "": 32946, + "": 32947, + "": 32785, + "": 32948, + "": 32949, + "": 32950, + "": 32951, + "": 32952, + "": 32953, + "": 32954, + "": 32955, + "": 32956, + "": 32957, + "": 32786, + "": 32958, + "": 32959, + "": 32960, + "": 32961, + "": 32962, + "": 32963, + "": 32964, + "": 32965, + "": 32966, + "": 32967, + "": 32787, + "": 32769, + "": 32968, + "": 32969, + "": 32970, + "": 32971, + "": 32972, + "": 32973, + "": 32974, + "": 32975, + "": 32976, + "": 32977, + "": 32788, + "": 32978, + "": 32979, + "": 32980, + "": 32981, + "": 32982, + "": 32983, + "": 32984, + "": 32985, + "": 32986, + "": 32987, + "": 32789, + "": 32988, + "": 32989, + "": 32990, + "": 32991, + "": 32992, + "": 32993, + "": 32994, + "": 32995, + "": 32996, + "": 32997, + "": 32790, + "": 32998, + "": 32999, + "": 33000, + "": 33001, + "": 33002, + "": 33003, + "": 33004, + "": 33005, + "": 33006, + "": 33007, + "": 32791, + "": 33008, + "": 33009, + "": 33010, + "": 33011, + "": 33012, + "": 33013, + "": 33014, + "": 33015, + "": 33016, + "": 33017, + "": 32792, + "": 33018, + "": 33019, + "": 33020, + "": 33021, + "": 33022, + "": 33023, + "": 32793, + "": 32794, + "": 32795, + "": 32796, + "": 32797, + "": 32770, + "": 32798, + "": 32799, + "": 32800, + "": 32801, + "": 32802, + "": 32803, + "": 32804, + "": 32805, + "": 32806, + "": 32807, + "": 32771, + "": 32808, + "": 32809, + "": 32810, + "": 32811, + "": 32812, + "": 32813, + "": 32814, + "": 32815, + "": 32816, + "": 32817, + "": 32772, + "": 32818, + "": 32819, + "": 32820, + "": 32821, + "": 32822, + "": 32823, + "": 32824, + "": 32825, + "": 32826, + "": 32827, + "": 32773, + "": 32828, + "": 32829, + "": 32830, + "": 32831, + "": 32832, + "": 32833, + "": 32834, + "": 32835, + "": 32836, + "": 32837, + "": 32774, + "": 32838, + "": 32839, + "": 32840, + "": 32841, + "": 32842, + "": 32843, + "": 32844, + "": 32845, + "": 32846, + "": 32847, + "": 32775, + "": 32848, + "": 32849, + "": 32850, + "": 32851, + "": 32852, + "": 32853, + "": 32854, + "": 32855, + "": 32856, + "": 32857, + "": 32776, + "": 32858, + "": 32859, + "": 32860, + "": 32861, + "": 32862, + "": 32863, + "": 32864, + "": 32865, + "": 32866, + "": 32867, + "": 32777 +} diff --git a/Ins/checkpoint-9678/added_tokens.json b/Ins/checkpoint-9678/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..d98719c951a52f071b6df8945c2261f26413dff1 --- /dev/null +++ b/Ins/checkpoint-9678/added_tokens.json @@ -0,0 +1,1026 @@ +{ + "": 32000, + "": 32100, + "": 32101, + "": 32102, + "": 32103, + "": 32104, + "": 32105, + "": 32106, + "": 32107, + "": 32108, + "": 32109, + "": 32010, + "": 32110, + "": 32111, + "": 32112, + "": 32113, + "": 32114, + "": 32115, + "": 32116, + "": 32117, + "": 32118, + "": 32119, + "": 32011, + "": 32120, + "": 32121, + "": 32122, + "": 32123, + "": 32124, + "": 32125, + "": 32126, + "": 32127, + "": 32128, + "": 32129, + "": 32012, + "": 32130, + "": 32131, + "": 32132, + "": 32133, + "": 32134, + "": 32135, + "": 32136, + "": 32137, + "": 32138, + "": 32139, + "": 32013, + "": 32140, + "": 32141, + "": 32142, + "": 32143, + "": 32144, + "": 32145, + "": 32146, + "": 32147, + "": 32148, + "": 32149, + "": 32014, + "": 32150, + "": 32151, + "": 32152, + "": 32153, + "": 32154, + "": 32155, + "": 32156, + "": 32157, + "": 32158, + "": 32159, + "": 32015, + "": 32160, + "": 32161, + "": 32162, + "": 32163, + "": 32164, + "": 32165, + "": 32166, + "": 32167, + "": 32168, + "": 32169, + "": 32016, + "": 32170, + "": 32171, + "": 32172, + "": 32173, + "": 32174, + "": 32175, + "": 32176, + "": 32177, + "": 32178, + "": 32179, + "": 32017, + "": 32180, + "": 32181, + "": 32182, + "": 32183, + "": 32184, + "": 32185, + "": 32186, + "": 32187, + "": 32188, + "": 32189, + "": 32018, + "": 32190, + "": 32191, + "": 32192, + "": 32193, + "": 32194, + "": 32195, + "": 32196, + "": 32197, + "": 32198, + "": 32199, + "": 32019, + "": 32001, + "": 32200, + "": 32201, + "": 32202, + "": 32203, + "": 32204, + "": 32205, + "": 32206, + "": 32207, + "": 32208, + "": 32209, + "": 32020, + "": 32210, + "": 32211, + "": 32212, + "": 32213, + "": 32214, + "": 32215, + "": 32216, + "": 32217, + "": 32218, + "": 32219, + "": 32021, + "": 32220, + "": 32221, + "": 32222, + "": 32223, + "": 32224, + "": 32225, + "": 32226, + "": 32227, + "": 32228, + "": 32229, + "": 32022, + "": 32230, + "": 32231, + "": 32232, + "": 32233, + "": 32234, + "": 32235, + "": 32236, + "": 32237, + "": 32238, + "": 32239, + "": 32023, + "": 32240, + "": 32241, + "": 32242, + "": 32243, + "": 32244, + "": 32245, + "": 32246, + "": 32247, + "": 32248, + "": 32249, + "": 32024, + "": 32250, + "": 32251, + "": 32252, + "": 32253, + "": 32254, + "": 32255, + "": 32025, + "": 32026, + "": 32027, + "": 32028, + "": 32029, + "": 32002, + "": 32030, + "": 32031, + "": 32032, + "": 32033, + "": 32034, + "": 32035, + "": 32036, + "": 32037, + "": 32038, + "": 32039, + "": 32003, + "": 32040, + "": 32041, + "": 32042, + "": 32043, + "": 32044, + "": 32045, + "": 32046, + "": 32047, + "": 32048, + "": 32049, + "": 32004, + "": 32050, + "": 32051, + "": 32052, + "": 32053, + "": 32054, + "": 32055, + "": 32056, + "": 32057, + "": 32058, + "": 32059, + "": 32005, + "": 32060, + "": 32061, + "": 32062, + "": 32063, + "": 32064, + "": 32065, + "": 32066, + "": 32067, + "": 32068, + "": 32069, + "": 32006, + "": 32070, + "": 32071, + "": 32072, + "": 32073, + "": 32074, + "": 32075, + "": 32076, + "": 32077, + "": 32078, + "": 32079, + "": 32007, + "": 32080, + "": 32081, + "": 32082, + "": 32083, + "": 32084, + "": 32085, + "": 32086, + "": 32087, + "": 32088, + "": 32089, + "": 32008, + "": 32090, + "": 32091, + "": 32092, + "": 32093, + "": 32094, + "": 32095, + "": 32096, + "": 32097, + "": 32098, + "": 32099, + "": 32009, + "": 32256, + "": 32356, + "": 32357, + "": 32358, + "": 32359, + "": 32360, + "": 32361, + "": 32362, + "": 32363, + "": 32364, + "": 32365, + "": 32266, + "": 32366, + "": 32367, + "": 32368, + "": 32369, + "": 32370, + "": 32371, + "": 32372, + "": 32373, + "": 32374, + "": 32375, + "": 32267, + "": 32376, + "": 32377, + "": 32378, + "": 32379, + "": 32380, + "": 32381, + "": 32382, + "": 32383, + "": 32384, + "": 32385, + "": 32268, + "": 32386, + "": 32387, + "": 32388, + "": 32389, + "": 32390, + "": 32391, + "": 32392, + "": 32393, + "": 32394, + "": 32395, + "": 32269, + "": 32396, + "": 32397, + "": 32398, + "": 32399, + "": 32400, + "": 32401, + "": 32402, + "": 32403, + "": 32404, + "": 32405, + "": 32270, + "": 32406, + "": 32407, + "": 32408, + "": 32409, + "": 32410, + "": 32411, + "": 32412, + "": 32413, + "": 32414, + "": 32415, + "": 32271, + "": 32416, + "": 32417, + "": 32418, + "": 32419, + "": 32420, + "": 32421, + "": 32422, + "": 32423, + "": 32424, + "": 32425, + "": 32272, + "": 32426, + "": 32427, + "": 32428, + "": 32429, + "": 32430, + "": 32431, + "": 32432, + "": 32433, + "": 32434, + "": 32435, + "": 32273, + "": 32436, + "": 32437, + "": 32438, + "": 32439, + "": 32440, + "": 32441, + "": 32442, + "": 32443, + "": 32444, + "": 32445, + "": 32274, + "": 32446, + "": 32447, + "": 32448, + "": 32449, + "": 32450, + "": 32451, + "": 32452, + "": 32453, + "": 32454, + "": 32455, + "": 32275, + "": 32257, + "": 32456, + "": 32457, + "": 32458, + "": 32459, + "": 32460, + "": 32461, + "": 32462, + "": 32463, + "": 32464, + "": 32465, + "": 32276, + "": 32466, + "": 32467, + "": 32468, + "": 32469, + "": 32470, + "": 32471, + "": 32472, + "": 32473, + "": 32474, + "": 32475, + "": 32277, + "": 32476, + "": 32477, + "": 32478, + "": 32479, + "": 32480, + "": 32481, + "": 32482, + "": 32483, + "": 32484, + "": 32485, + "": 32278, + "": 32486, + "": 32487, + "": 32488, + "": 32489, + "": 32490, + "": 32491, + "": 32492, + "": 32493, + "": 32494, + "": 32495, + "": 32279, + "": 32496, + "": 32497, + "": 32498, + "": 32499, + "": 32500, + "": 32501, + "": 32502, + "": 32503, + "": 32504, + "": 32505, + "": 32280, + "": 32506, + "": 32507, + "": 32508, + "": 32509, + "": 32510, + "": 32511, + "": 32281, + "": 32282, + "": 32283, + "": 32284, + "": 32285, + "": 32258, + "": 32286, + "": 32287, + "": 32288, + "": 32289, + "": 32290, + "": 32291, + "": 32292, + "": 32293, + "": 32294, + "": 32295, + "": 32259, + "": 32296, + "": 32297, + "": 32298, + "": 32299, + "": 32300, + "": 32301, + "": 32302, + "": 32303, + "": 32304, + "": 32305, + "": 32260, + "": 32306, + "": 32307, + "": 32308, + "": 32309, + "": 32310, + "": 32311, + "": 32312, + "": 32313, + "": 32314, + "": 32315, + "": 32261, + "": 32316, + "": 32317, + "": 32318, + "": 32319, + "": 32320, + "": 32321, + "": 32322, + "": 32323, + "": 32324, + "": 32325, + "": 32262, + "": 32326, + "": 32327, + "": 32328, + "": 32329, + "": 32330, + "": 32331, + "": 32332, + "": 32333, + "": 32334, + "": 32335, + "": 32263, + "": 32336, + "": 32337, + "": 32338, + "": 32339, + "": 32340, + "": 32341, + "": 32342, + "": 32343, + "": 32344, + "": 32345, + "": 32264, + "": 32346, + "": 32347, + "": 32348, + "": 32349, + "": 32350, + "": 32351, + "": 32352, + "": 32353, + "": 32354, + "": 32355, + "": 32265, + "": 32512, + "": 32612, + "": 32613, + "": 32614, + "": 32615, + "": 32616, + "": 32617, + "": 32618, + "": 32619, + "": 32620, + "": 32621, + "": 32522, + "": 32622, + "": 32623, + "": 32624, + "": 32625, + "": 32626, + "": 32627, + "": 32628, + "": 32629, + "": 32630, + "": 32631, + "": 32523, + "": 32632, + "": 32633, + "": 32634, + "": 32635, + "": 32636, + "": 32637, + "": 32638, + "": 32639, + "": 32640, + "": 32641, + "": 32524, + "": 32642, + "": 32643, + "": 32644, + "": 32645, + "": 32646, + "": 32647, + "": 32648, + "": 32649, + "": 32650, + "": 32651, + "": 32525, + "": 32652, + "": 32653, + "": 32654, + "": 32655, + "": 32656, + "": 32657, + "": 32658, + "": 32659, + "": 32660, + "": 32661, + "": 32526, + "": 32662, + "": 32663, + "": 32664, + "": 32665, + "": 32666, + "": 32667, + "": 32668, + "": 32669, + "": 32670, + "": 32671, + "": 32527, + "": 32672, + "": 32673, + "": 32674, + "": 32675, + "": 32676, + "": 32677, + "": 32678, + "": 32679, + "": 32680, + "": 32681, + "": 32528, + "": 32682, + "": 32683, + "": 32684, + "": 32685, + "": 32686, + "": 32687, + "": 32688, + "": 32689, + "": 32690, + "": 32691, + "": 32529, + "": 32692, + "": 32693, + "": 32694, + "": 32695, + "": 32696, + "": 32697, + "": 32698, + "": 32699, + "": 32700, + "": 32701, + "": 32530, + "": 32702, + "": 32703, + "": 32704, + "": 32705, + "": 32706, + "": 32707, + "": 32708, + "": 32709, + "": 32710, + "": 32711, + "": 32531, + "": 32513, + "": 32712, + "": 32713, + "": 32714, + "": 32715, + "": 32716, + "": 32717, + "": 32718, + "": 32719, + "": 32720, + "": 32721, + "": 32532, + "": 32722, + "": 32723, + "": 32724, + "": 32725, + "": 32726, + "": 32727, + "": 32728, + "": 32729, + "": 32730, + "": 32731, + "": 32533, + "": 32732, + "": 32733, + "": 32734, + "": 32735, + "": 32736, + "": 32737, + "": 32738, + "": 32739, + "": 32740, + "": 32741, + "": 32534, + "": 32742, + "": 32743, + "": 32744, + "": 32745, + "": 32746, + "": 32747, + "": 32748, + "": 32749, + "": 32750, + "": 32751, + "": 32535, + "": 32752, + "": 32753, + "": 32754, + "": 32755, + "": 32756, + "": 32757, + "": 32758, + "": 32759, + "": 32760, + "": 32761, + "": 32536, + "": 32762, + "": 32763, + "": 32764, + "": 32765, + "": 32766, + "": 32767, + "": 32537, + "": 32538, + "": 32539, + "": 32540, + "": 32541, + "": 32514, + "": 32542, + "": 32543, + "": 32544, + "": 32545, + "": 32546, + "": 32547, + "": 32548, + "": 32549, + "": 32550, + "": 32551, + "": 32515, + "": 32552, + "": 32553, + "": 32554, + "": 32555, + "": 32556, + "": 32557, + "": 32558, + "": 32559, + "": 32560, + "": 32561, + "": 32516, + "": 32562, + "": 32563, + "": 32564, + "": 32565, + "": 32566, + "": 32567, + "": 32568, + "": 32569, + "": 32570, + "": 32571, + "": 32517, + "": 32572, + "": 32573, + "": 32574, + "": 32575, + "": 32576, + "": 32577, + "": 32578, + "": 32579, + "": 32580, + "": 32581, + "": 32518, + "": 32582, + "": 32583, + "": 32584, + "": 32585, + "": 32586, + "": 32587, + "": 32588, + "": 32589, + "": 32590, + "": 32591, + "": 32519, + "": 32592, + "": 32593, + "": 32594, + "": 32595, + "": 32596, + "": 32597, + "": 32598, + "": 32599, + "": 32600, + "": 32601, + "": 32520, + "": 32602, + "": 32603, + "": 32604, + "": 32605, + "": 32606, + "": 32607, + "": 32608, + "": 32609, + "": 32610, + "": 32611, + "": 32521, + "": 32768, + "": 32868, + "": 32869, + "": 32870, + "": 32871, + "": 32872, + "": 32873, + "": 32874, + "": 32875, + "": 32876, + "": 32877, + "": 32778, + "": 32878, + "": 32879, + "": 32880, + "": 32881, + "": 32882, + "": 32883, + "": 32884, + "": 32885, + "": 32886, + "": 32887, + "": 32779, + "": 32888, + "": 32889, + "": 32890, + "": 32891, + "": 32892, + "": 32893, + "": 32894, + "": 32895, + "": 32896, + "": 32897, + "": 32780, + "": 32898, + "": 32899, + "": 32900, + "": 32901, + "": 32902, + "": 32903, + "": 32904, + "": 32905, + "": 32906, + "": 32907, + "": 32781, + "": 32908, + "": 32909, + "": 32910, + "": 32911, + "": 32912, + "": 32913, + "": 32914, + "": 32915, + "": 32916, + "": 32917, + "": 32782, + "": 32918, + "": 32919, + "": 32920, + "": 32921, + "": 32922, + "": 32923, + "": 32924, + "": 32925, + "": 32926, + "": 32927, + "": 32783, + "": 32928, + "": 32929, + "": 32930, + "": 32931, + "": 32932, + "": 32933, + "": 32934, + "": 32935, + "": 32936, + "": 32937, + "": 32784, + "": 32938, + "": 32939, + "": 32940, + "": 32941, + "": 32942, + "": 32943, + "": 32944, + "": 32945, + "": 32946, + "": 32947, + "": 32785, + "": 32948, + "": 32949, + "": 32950, + "": 32951, + "": 32952, + "": 32953, + "": 32954, + "": 32955, + "": 32956, + "": 32957, + "": 32786, + "": 32958, + "": 32959, + "": 32960, + "": 32961, + "": 32962, + "": 32963, + "": 32964, + "": 32965, + "": 32966, + "": 32967, + "": 32787, + "": 32769, + "": 32968, + "": 32969, + "": 32970, + "": 32971, + "": 32972, + "": 32973, + "": 32974, + "": 32975, + "": 32976, + "": 32977, + "": 32788, + "": 32978, + "": 32979, + "": 32980, + "": 32981, + "": 32982, + "": 32983, + "": 32984, + "": 32985, + "": 32986, + "": 32987, + "": 32789, + "": 32988, + "": 32989, + "": 32990, + "": 32991, + "": 32992, + "": 32993, + "": 32994, + "": 32995, + "": 32996, + "": 32997, + "": 32790, + "": 32998, + "": 32999, + "": 33000, + "": 33001, + "": 33002, + "": 33003, + "": 33004, + "": 33005, + "": 33006, + "": 33007, + "": 32791, + "": 33008, + "": 33009, + "": 33010, + "": 33011, + "": 33012, + "": 33013, + "": 33014, + "": 33015, + "": 33016, + "": 33017, + "": 32792, + "": 33018, + "": 33019, + "": 33020, + "": 33021, + "": 33022, + "": 33023, + "": 32793, + "": 32794, + "": 32795, + "": 32796, + "": 32797, + "": 32770, + "": 32798, + "": 32799, + "": 32800, + "": 32801, + "": 32802, + "": 32803, + "": 32804, + "": 32805, + "": 32806, + "": 32807, + "": 32771, + "": 32808, + "": 32809, + "": 32810, + "": 32811, + "": 32812, + "": 32813, + "": 32814, + "": 32815, + "": 32816, + "": 32817, + "": 32772, + "": 32818, + "": 32819, + "": 32820, + "": 32821, + "": 32822, + "": 32823, + "": 32824, + "": 32825, + "": 32826, + "": 32827, + "": 32773, + "": 32828, + "": 32829, + "": 32830, + "": 32831, + "": 32832, + "": 32833, + "": 32834, + "": 32835, + "": 32836, + "": 32837, + "": 32774, + "": 32838, + "": 32839, + "": 32840, + "": 32841, + "": 32842, + "": 32843, + "": 32844, + "": 32845, + "": 32846, + "": 32847, + "": 32775, + "": 32848, + "": 32849, + "": 32850, + "": 32851, + "": 32852, + "": 32853, + "": 32854, + "": 32855, + "": 32856, + "": 32857, + "": 32776, + "": 32858, + "": 32859, + "": 32860, + "": 32861, + "": 32862, + "": 32863, + "": 32864, + "": 32865, + "": 32866, + "": 32867, + "": 32777 +} diff --git a/Ins/checkpoint-9678/latest b/Ins/checkpoint-9678/latest new file mode 100644 index 0000000000000000000000000000000000000000..5849afe6037519d81ecaf7d25328303cdf2b5ca2 --- /dev/null +++ b/Ins/checkpoint-9678/latest @@ -0,0 +1 @@ +global_step9678 \ No newline at end of file diff --git a/Ins/checkpoint-9678/model.safetensors.index.json b/Ins/checkpoint-9678/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..92673112580ef150675df9b47e07c4f8dc4e4bd6 --- /dev/null +++ b/Ins/checkpoint-9678/model.safetensors.index.json @@ -0,0 +1,780 @@ +{ + "metadata": { + "total_size": 14119477056 + }, + "weight_map": { + "model.base_model.model.lm_head.modules_to_save.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.lm_head.original_module.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.embed_tokens.modules_to_save.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.embed_tokens.original_module.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.0.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.1.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.10.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.11.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.12.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.13.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.14.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.15.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.16.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.17.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.18.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.19.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.2.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.20.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.21.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.down_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.down_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.gate_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.gate_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.gate_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.up_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.up_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.mlp.up_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.k_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.k_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.k_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.o_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.o_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.v_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.v_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.22.self_attn.v_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.q_proj.base_layer.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.q_proj.lora_A.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.q_proj.lora_B.default.weight": "model-00002-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.23.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.24.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.25.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.26.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.27.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.28.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.29.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.3.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.30.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.down_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.down_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.gate_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.gate_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.gate_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.up_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.up_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.mlp.up_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.k_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.k_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.k_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.o_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.o_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.q_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.q_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.q_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.v_proj.base_layer.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.v_proj.lora_A.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.31.self_attn.v_proj.lora_B.default.weight": "model-00003-of-00003.safetensors", + "model.base_model.model.model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.4.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.5.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.6.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.7.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.8.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.down_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.down_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.down_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.gate_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.gate_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.gate_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.up_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.up_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.mlp.up_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.k_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.k_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.k_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.o_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.o_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.q_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.q_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.q_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.v_proj.base_layer.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.v_proj.lora_A.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.layers.9.self_attn.v_proj.lora_B.default.weight": "model-00001-of-00003.safetensors", + "model.base_model.model.model.norm.weight": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.1.bias": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.1.weight": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.10.bias": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.10.weight": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.13.bias": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.13.weight": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.16.bias": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.16.weight": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.19.bias": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.19.weight": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.4.bias": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.4.weight": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.7.bias": "model-00003-of-00003.safetensors", + "rqvae.decoder.mlp_layers.7.weight": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.1.bias": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.1.weight": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.10.bias": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.10.weight": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.13.bias": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.13.weight": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.16.bias": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.16.weight": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.19.bias": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.19.weight": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.4.bias": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.4.weight": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.7.bias": "model-00003-of-00003.safetensors", + "rqvae.encoder.mlp_layers.7.weight": "model-00003-of-00003.safetensors", + "rqvae.rq.vq_layers.0.embedding.weight": "model-00003-of-00003.safetensors", + "rqvae.rq.vq_layers.1.embedding.weight": "model-00003-of-00003.safetensors", + "rqvae.rq.vq_layers.2.embedding.weight": "model-00003-of-00003.safetensors", + "rqvae.rq.vq_layers.3.embedding.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/Ins/checkpoint-9678/trainer_state.json b/Ins/checkpoint-9678/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3799392941b9f229c051c7d1506574131934e712 --- /dev/null +++ b/Ins/checkpoint-9678/trainer_state.json @@ -0,0 +1,6810 @@ +{ + "best_metric": 2.729660987854004, + "best_model_checkpoint": "./Ins/checkpoint-9678", + "epoch": 1.0, + "eval_steps": 1000, + "global_step": 9678, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0010332713370531101, + "grad_norm": 9.612689971923828, + "learning_rate": 5.1546391752577315e-05, + "loss": 4.2162, + "step": 10 + }, + { + "epoch": 0.0020665426741062203, + "grad_norm": 6.56850004196167, + "learning_rate": 0.00010309278350515463, + "loss": 3.388, + "step": 20 + }, + { + "epoch": 0.0030998140111593306, + "grad_norm": 4.049291610717773, + "learning_rate": 0.00015463917525773197, + "loss": 2.7081, + "step": 30 + }, + { + "epoch": 0.0041330853482124405, + "grad_norm": 2.604156970977783, + "learning_rate": 0.00020618556701030926, + "loss": 2.8688, + "step": 40 + }, + { + "epoch": 0.0051663566852655505, + "grad_norm": 1.8748372793197632, + "learning_rate": 0.00025773195876288655, + "loss": 2.6251, + "step": 50 + }, + { + "epoch": 0.006199628022318661, + "grad_norm": 2.638007164001465, + "learning_rate": 0.00030927835051546395, + "loss": 2.5802, + "step": 60 + }, + { + "epoch": 0.007232899359371771, + "grad_norm": 1.8762423992156982, + "learning_rate": 0.00036082474226804123, + "loss": 2.4978, + "step": 70 + }, + { + "epoch": 0.008266170696424881, + "grad_norm": 1.7499455213546753, + "learning_rate": 0.0004123711340206185, + "loss": 2.4351, + "step": 80 + }, + { + "epoch": 0.009299442033477991, + "grad_norm": 1.4334783554077148, + "learning_rate": 0.0004639175257731959, + "loss": 2.4811, + "step": 90 + }, + { + "epoch": 0.010332713370531101, + "grad_norm": 1.4335546493530273, + "learning_rate": 0.0004999998790431266, + "loss": 2.3653, + "step": 100 + }, + { + "epoch": 0.01136598470758421, + "grad_norm": 1.379575490951538, + "learning_rate": 0.0004999977287019656, + "loss": 2.2769, + "step": 110 + }, + { + "epoch": 0.012399256044637322, + "grad_norm": 1.357542872428894, + "learning_rate": 0.0004999928904568952, + "loss": 2.1347, + "step": 120 + }, + { + "epoch": 0.013432527381690432, + "grad_norm": 1.6458231210708618, + "learning_rate": 0.0004999853643599349, + "loss": 2.1943, + "step": 130 + }, + { + "epoch": 0.014465798718743542, + "grad_norm": 1.7004691362380981, + "learning_rate": 0.0004999751504920031, + "loss": 2.204, + "step": 140 + }, + { + "epoch": 0.015499070055796652, + "grad_norm": 1.3042373657226562, + "learning_rate": 0.0004999622489629164, + "loss": 2.0817, + "step": 150 + }, + { + "epoch": 0.016532341392849762, + "grad_norm": 1.5508415699005127, + "learning_rate": 0.0004999466599113884, + "loss": 2.115, + "step": 160 + }, + { + "epoch": 0.017565612729902872, + "grad_norm": 1.3333240747451782, + "learning_rate": 0.0004999283835050278, + "loss": 2.1762, + "step": 170 + }, + { + "epoch": 0.018598884066955982, + "grad_norm": 1.3132269382476807, + "learning_rate": 0.0004999074199403377, + "loss": 2.0493, + "step": 180 + }, + { + "epoch": 0.019632155404009092, + "grad_norm": 1.250373363494873, + "learning_rate": 0.000499883769442712, + "loss": 2.2234, + "step": 190 + }, + { + "epoch": 0.020665426741062202, + "grad_norm": 1.2733889818191528, + "learning_rate": 0.0004998574322664343, + "loss": 1.9751, + "step": 200 + }, + { + "epoch": 0.02169869807811531, + "grad_norm": 0.9156686663627625, + "learning_rate": 0.0004998284086946743, + "loss": 2.1314, + "step": 210 + }, + { + "epoch": 0.02273196941516842, + "grad_norm": 1.0921283960342407, + "learning_rate": 0.0004997966990394851, + "loss": 1.8402, + "step": 220 + }, + { + "epoch": 0.023765240752221535, + "grad_norm": 1.2582165002822876, + "learning_rate": 0.0004997623036417998, + "loss": 2.1236, + "step": 230 + }, + { + "epoch": 0.024798512089274645, + "grad_norm": 1.1085331439971924, + "learning_rate": 0.0004997252228714278, + "loss": 2.0778, + "step": 240 + }, + { + "epoch": 0.025831783426327755, + "grad_norm": 1.096815824508667, + "learning_rate": 0.0004996854571270512, + "loss": 2.146, + "step": 250 + }, + { + "epoch": 0.026865054763380865, + "grad_norm": 1.0710986852645874, + "learning_rate": 0.0004996430068362196, + "loss": 1.9776, + "step": 260 + }, + { + "epoch": 0.027898326100433975, + "grad_norm": 1.4760586023330688, + "learning_rate": 0.0004995978724553464, + "loss": 1.9536, + "step": 270 + }, + { + "epoch": 0.028931597437487085, + "grad_norm": 1.3000599145889282, + "learning_rate": 0.0004995500544697036, + "loss": 1.7954, + "step": 280 + }, + { + "epoch": 0.029964868774540195, + "grad_norm": 1.2158455848693848, + "learning_rate": 0.0004994995533934163, + "loss": 1.8429, + "step": 290 + }, + { + "epoch": 0.030998140111593304, + "grad_norm": 1.1453113555908203, + "learning_rate": 0.0004994463697694579, + "loss": 2.0769, + "step": 300 + }, + { + "epoch": 0.032031411448646414, + "grad_norm": 0.7675238847732544, + "learning_rate": 0.0004993905041696433, + "loss": 1.903, + "step": 310 + }, + { + "epoch": 0.033064682785699524, + "grad_norm": 1.1099224090576172, + "learning_rate": 0.0004993319571946235, + "loss": 1.9536, + "step": 320 + }, + { + "epoch": 0.034097954122752634, + "grad_norm": 1.617551326751709, + "learning_rate": 0.0004992707294738786, + "loss": 1.9095, + "step": 330 + }, + { + "epoch": 0.035131225459805744, + "grad_norm": 1.486296534538269, + "learning_rate": 0.0004992068216657115, + "loss": 1.8174, + "step": 340 + }, + { + "epoch": 0.036164496796858854, + "grad_norm": 1.4763352870941162, + "learning_rate": 0.0004991402344572409, + "loss": 1.7963, + "step": 350 + }, + { + "epoch": 0.037197768133911964, + "grad_norm": 1.2317055463790894, + "learning_rate": 0.0004990709685643932, + "loss": 1.9319, + "step": 360 + }, + { + "epoch": 0.038231039470965074, + "grad_norm": 1.074246883392334, + "learning_rate": 0.0004989990247318954, + "loss": 1.9447, + "step": 370 + }, + { + "epoch": 0.039264310808018184, + "grad_norm": 1.2657012939453125, + "learning_rate": 0.0004989244037332671, + "loss": 1.7053, + "step": 380 + }, + { + "epoch": 0.040297582145071294, + "grad_norm": 1.3143869638442993, + "learning_rate": 0.0004988471063708116, + "loss": 1.8009, + "step": 390 + }, + { + "epoch": 0.041330853482124404, + "grad_norm": 1.381238579750061, + "learning_rate": 0.0004987671334756084, + "loss": 1.8229, + "step": 400 + }, + { + "epoch": 0.042364124819177514, + "grad_norm": 1.2067453861236572, + "learning_rate": 0.0004986844859075028, + "loss": 1.7401, + "step": 410 + }, + { + "epoch": 0.04339739615623062, + "grad_norm": 1.2656382322311401, + "learning_rate": 0.0004985991645550979, + "loss": 1.6774, + "step": 420 + }, + { + "epoch": 0.04443066749328373, + "grad_norm": 0.9306573271751404, + "learning_rate": 0.0004985111703357444, + "loss": 1.7884, + "step": 430 + }, + { + "epoch": 0.04546393883033684, + "grad_norm": 0.9404155611991882, + "learning_rate": 0.0004984205041955309, + "loss": 1.6522, + "step": 440 + }, + { + "epoch": 0.04649721016738995, + "grad_norm": 1.2634170055389404, + "learning_rate": 0.0004983271671092738, + "loss": 1.7681, + "step": 450 + }, + { + "epoch": 0.04753048150444307, + "grad_norm": 1.6461280584335327, + "learning_rate": 0.0004982311600805066, + "loss": 1.8906, + "step": 460 + }, + { + "epoch": 0.04856375284149618, + "grad_norm": 1.242746114730835, + "learning_rate": 0.0004981324841414695, + "loss": 1.8049, + "step": 470 + }, + { + "epoch": 0.04959702417854929, + "grad_norm": 1.0817033052444458, + "learning_rate": 0.0004980311403530978, + "loss": 1.6272, + "step": 480 + }, + { + "epoch": 0.0506302955156024, + "grad_norm": 1.2890691757202148, + "learning_rate": 0.0004979271298050107, + "loss": 1.6988, + "step": 490 + }, + { + "epoch": 0.05166356685265551, + "grad_norm": 1.0929986238479614, + "learning_rate": 0.0004978204536155003, + "loss": 1.728, + "step": 500 + }, + { + "epoch": 0.05269683818970862, + "grad_norm": 1.2483139038085938, + "learning_rate": 0.0004977111129315177, + "loss": 1.5616, + "step": 510 + }, + { + "epoch": 0.05373010952676173, + "grad_norm": 1.0845454931259155, + "learning_rate": 0.0004975991089286632, + "loss": 1.5409, + "step": 520 + }, + { + "epoch": 0.05476338086381484, + "grad_norm": 1.4180911779403687, + "learning_rate": 0.0004974844428111715, + "loss": 1.7888, + "step": 530 + }, + { + "epoch": 0.05579665220086795, + "grad_norm": 1.0860427618026733, + "learning_rate": 0.0004973671158118998, + "loss": 1.6986, + "step": 540 + }, + { + "epoch": 0.05682992353792106, + "grad_norm": 1.1166560649871826, + "learning_rate": 0.0004972471291923143, + "loss": 1.8097, + "step": 550 + }, + { + "epoch": 0.05786319487497417, + "grad_norm": 1.2548375129699707, + "learning_rate": 0.0004971244842424768, + "loss": 1.6169, + "step": 560 + }, + { + "epoch": 0.05889646621202728, + "grad_norm": 0.991226851940155, + "learning_rate": 0.0004969991822810307, + "loss": 1.6965, + "step": 570 + }, + { + "epoch": 0.05992973754908039, + "grad_norm": 1.1663395166397095, + "learning_rate": 0.0004968712246551868, + "loss": 1.6587, + "step": 580 + }, + { + "epoch": 0.0609630088861335, + "grad_norm": 1.3420109748840332, + "learning_rate": 0.0004967406127407086, + "loss": 1.6837, + "step": 590 + }, + { + "epoch": 0.06199628022318661, + "grad_norm": 1.0421655178070068, + "learning_rate": 0.0004966073479418982, + "loss": 1.637, + "step": 600 + }, + { + "epoch": 0.06302955156023972, + "grad_norm": 1.2786924839019775, + "learning_rate": 0.0004964714316915803, + "loss": 1.5727, + "step": 610 + }, + { + "epoch": 0.06406282289729283, + "grad_norm": 1.507780909538269, + "learning_rate": 0.0004963328654510877, + "loss": 1.6056, + "step": 620 + }, + { + "epoch": 0.06509609423434594, + "grad_norm": 1.0936928987503052, + "learning_rate": 0.0004961916507102447, + "loss": 1.5742, + "step": 630 + }, + { + "epoch": 0.06612936557139905, + "grad_norm": 1.2347360849380493, + "learning_rate": 0.0004960477889873517, + "loss": 1.6692, + "step": 640 + }, + { + "epoch": 0.06716263690845216, + "grad_norm": 1.4181699752807617, + "learning_rate": 0.0004959012818291688, + "loss": 1.6412, + "step": 650 + }, + { + "epoch": 0.06819590824550527, + "grad_norm": 1.3150395154953003, + "learning_rate": 0.000495752130810899, + "loss": 1.4951, + "step": 660 + }, + { + "epoch": 0.06922917958255838, + "grad_norm": 1.5858718156814575, + "learning_rate": 0.0004956003375361712, + "loss": 1.6635, + "step": 670 + }, + { + "epoch": 0.07026245091961149, + "grad_norm": 1.1488349437713623, + "learning_rate": 0.0004954459036370231, + "loss": 1.6101, + "step": 680 + }, + { + "epoch": 0.0712957222566646, + "grad_norm": 1.0433565378189087, + "learning_rate": 0.0004952888307738839, + "loss": 1.7019, + "step": 690 + }, + { + "epoch": 0.07232899359371771, + "grad_norm": 1.2093909978866577, + "learning_rate": 0.000495129120635556, + "loss": 1.6477, + "step": 700 + }, + { + "epoch": 0.07336226493077082, + "grad_norm": 1.1268610954284668, + "learning_rate": 0.0004949667749391967, + "loss": 1.5262, + "step": 710 + }, + { + "epoch": 0.07439553626782393, + "grad_norm": 1.0410875082015991, + "learning_rate": 0.0004948017954303007, + "loss": 1.7201, + "step": 720 + }, + { + "epoch": 0.07542880760487704, + "grad_norm": 1.5368582010269165, + "learning_rate": 0.0004946341838826803, + "loss": 1.7024, + "step": 730 + }, + { + "epoch": 0.07646207894193015, + "grad_norm": 0.9913454651832581, + "learning_rate": 0.0004944639420984469, + "loss": 1.4697, + "step": 740 + }, + { + "epoch": 0.07749535027898326, + "grad_norm": 1.1712089776992798, + "learning_rate": 0.0004942910719079912, + "loss": 1.4866, + "step": 750 + }, + { + "epoch": 0.07852862161603637, + "grad_norm": 1.5118815898895264, + "learning_rate": 0.0004941155751699639, + "loss": 1.6489, + "step": 760 + }, + { + "epoch": 0.07956189295308948, + "grad_norm": 0.9319295883178711, + "learning_rate": 0.0004939374537712558, + "loss": 1.5053, + "step": 770 + }, + { + "epoch": 0.08059516429014259, + "grad_norm": 2.0669829845428467, + "learning_rate": 0.0004937567096269769, + "loss": 1.6412, + "step": 780 + }, + { + "epoch": 0.0816284356271957, + "grad_norm": 1.876446008682251, + "learning_rate": 0.0004935733446804367, + "loss": 1.7186, + "step": 790 + }, + { + "epoch": 0.08266170696424881, + "grad_norm": 1.0011149644851685, + "learning_rate": 0.0004933873609031224, + "loss": 1.4197, + "step": 800 + }, + { + "epoch": 0.08369497830130192, + "grad_norm": 84.56254577636719, + "learning_rate": 0.000493198760294678, + "loss": 2.006, + "step": 810 + }, + { + "epoch": 0.08472824963835503, + "grad_norm": 9.466862678527832, + "learning_rate": 0.0004930075448828836, + "loss": 3.6223, + "step": 820 + }, + { + "epoch": 0.08576152097540814, + "grad_norm": 1.8478171825408936, + "learning_rate": 0.0004928137167236323, + "loss": 1.8625, + "step": 830 + }, + { + "epoch": 0.08679479231246125, + "grad_norm": 1.6235121488571167, + "learning_rate": 0.0004926172779009089, + "loss": 1.6312, + "step": 840 + }, + { + "epoch": 0.08782806364951436, + "grad_norm": 1.0582070350646973, + "learning_rate": 0.0004924182305267672, + "loss": 1.4884, + "step": 850 + }, + { + "epoch": 0.08886133498656747, + "grad_norm": 1.6170542240142822, + "learning_rate": 0.0004922165767413077, + "loss": 1.4772, + "step": 860 + }, + { + "epoch": 0.08989460632362058, + "grad_norm": 1.1165497303009033, + "learning_rate": 0.0004920123187126539, + "loss": 1.396, + "step": 870 + }, + { + "epoch": 0.09092787766067369, + "grad_norm": 1.5052683353424072, + "learning_rate": 0.0004918054586369299, + "loss": 1.7157, + "step": 880 + }, + { + "epoch": 0.0919611489977268, + "grad_norm": 1.5653349161148071, + "learning_rate": 0.0004915959987382355, + "loss": 1.5859, + "step": 890 + }, + { + "epoch": 0.0929944203347799, + "grad_norm": 1.4743231534957886, + "learning_rate": 0.0004913839412686238, + "loss": 1.5155, + "step": 900 + }, + { + "epoch": 0.09402769167183303, + "grad_norm": 1.931356430053711, + "learning_rate": 0.0004911692885080757, + "loss": 1.6259, + "step": 910 + }, + { + "epoch": 0.09506096300888614, + "grad_norm": 1.209402322769165, + "learning_rate": 0.0004909520427644762, + "loss": 1.544, + "step": 920 + }, + { + "epoch": 0.09609423434593925, + "grad_norm": 1.3506582975387573, + "learning_rate": 0.0004907322063735895, + "loss": 1.6404, + "step": 930 + }, + { + "epoch": 0.09712750568299236, + "grad_norm": 1.0187022686004639, + "learning_rate": 0.0004905097816990331, + "loss": 1.4471, + "step": 940 + }, + { + "epoch": 0.09816077702004547, + "grad_norm": 1.3293033838272095, + "learning_rate": 0.0004902847711322535, + "loss": 1.5233, + "step": 950 + }, + { + "epoch": 0.09919404835709858, + "grad_norm": 1.0673298835754395, + "learning_rate": 0.0004900571770924999, + "loss": 1.4875, + "step": 960 + }, + { + "epoch": 0.10022731969415169, + "grad_norm": 1.2059558629989624, + "learning_rate": 0.000489827002026798, + "loss": 1.5143, + "step": 970 + }, + { + "epoch": 0.1012605910312048, + "grad_norm": 1.5264915227890015, + "learning_rate": 0.0004895942484099241, + "loss": 1.3904, + "step": 980 + }, + { + "epoch": 0.10229386236825791, + "grad_norm": 1.055259108543396, + "learning_rate": 0.0004893589187443786, + "loss": 1.5727, + "step": 990 + }, + { + "epoch": 0.10332713370531102, + "grad_norm": 1.3388723134994507, + "learning_rate": 0.0004891210155603585, + "loss": 1.3997, + "step": 1000 + }, + { + "epoch": 0.10436040504236413, + "grad_norm": 1.146324872970581, + "learning_rate": 0.0004888805414157304, + "loss": 1.5489, + "step": 1010 + }, + { + "epoch": 0.10539367637941724, + "grad_norm": 1.446535348892212, + "learning_rate": 0.0004886374988960036, + "loss": 1.5821, + "step": 1020 + }, + { + "epoch": 0.10642694771647035, + "grad_norm": 1.3649553060531616, + "learning_rate": 0.0004883918906143016, + "loss": 1.4677, + "step": 1030 + }, + { + "epoch": 0.10746021905352346, + "grad_norm": 1.2442352771759033, + "learning_rate": 0.00048814371921133417, + "loss": 1.4691, + "step": 1040 + }, + { + "epoch": 0.10849349039057657, + "grad_norm": 1.2473433017730713, + "learning_rate": 0.00048789298735536904, + "loss": 1.6451, + "step": 1050 + }, + { + "epoch": 0.10952676172762968, + "grad_norm": 1.7001152038574219, + "learning_rate": 0.0004876396977422033, + "loss": 1.6799, + "step": 1060 + }, + { + "epoch": 0.11056003306468279, + "grad_norm": 1.1802128553390503, + "learning_rate": 0.00048738385309513434, + "loss": 1.4816, + "step": 1070 + }, + { + "epoch": 0.1115933044017359, + "grad_norm": 1.1580235958099365, + "learning_rate": 0.0004871254561649303, + "loss": 1.681, + "step": 1080 + }, + { + "epoch": 0.11262657573878901, + "grad_norm": 1.2179391384124756, + "learning_rate": 0.000486864509729801, + "loss": 1.507, + "step": 1090 + }, + { + "epoch": 0.11365984707584212, + "grad_norm": 1.1689331531524658, + "learning_rate": 0.00048660101659536763, + "loss": 1.276, + "step": 1100 + }, + { + "epoch": 0.11469311841289523, + "grad_norm": 1.1789075136184692, + "learning_rate": 0.0004863349795946329, + "loss": 1.5065, + "step": 1110 + }, + { + "epoch": 0.11572638974994834, + "grad_norm": 0.98433917760849, + "learning_rate": 0.00048606640158795034, + "loss": 1.286, + "step": 1120 + }, + { + "epoch": 0.11675966108700145, + "grad_norm": 0.9510318636894226, + "learning_rate": 0.0004857952854629938, + "loss": 1.4637, + "step": 1130 + }, + { + "epoch": 0.11779293242405456, + "grad_norm": 1.0436633825302124, + "learning_rate": 0.0004855216341347259, + "loss": 1.4925, + "step": 1140 + }, + { + "epoch": 0.11882620376110767, + "grad_norm": 1.4930212497711182, + "learning_rate": 0.0004852454505453674, + "loss": 1.3586, + "step": 1150 + }, + { + "epoch": 0.11985947509816078, + "grad_norm": 0.8224968910217285, + "learning_rate": 0.00048496673766436517, + "loss": 1.5212, + "step": 1160 + }, + { + "epoch": 0.12089274643521389, + "grad_norm": 1.2815650701522827, + "learning_rate": 0.00048468549848835996, + "loss": 1.4901, + "step": 1170 + }, + { + "epoch": 0.121926017772267, + "grad_norm": 1.2067404985427856, + "learning_rate": 0.000484401736041155, + "loss": 1.5946, + "step": 1180 + }, + { + "epoch": 0.12295928910932011, + "grad_norm": 1.530190348625183, + "learning_rate": 0.0004841154533736827, + "loss": 1.5052, + "step": 1190 + }, + { + "epoch": 0.12399256044637322, + "grad_norm": 1.2579395771026611, + "learning_rate": 0.0004838266535639722, + "loss": 1.4186, + "step": 1200 + }, + { + "epoch": 0.12502583178342633, + "grad_norm": 1.5573033094406128, + "learning_rate": 0.00048353533971711625, + "loss": 1.3459, + "step": 1210 + }, + { + "epoch": 0.12605910312047944, + "grad_norm": 1.2861477136611938, + "learning_rate": 0.0004832415149652378, + "loss": 1.4907, + "step": 1220 + }, + { + "epoch": 0.12709237445753255, + "grad_norm": 0.8996832370758057, + "learning_rate": 0.00048294518246745643, + "loss": 1.3579, + "step": 1230 + }, + { + "epoch": 0.12812564579458566, + "grad_norm": 1.4587196111679077, + "learning_rate": 0.0004826463454098542, + "loss": 1.3771, + "step": 1240 + }, + { + "epoch": 0.12915891713163877, + "grad_norm": 1.1128901243209839, + "learning_rate": 0.00048234500700544144, + "loss": 1.3366, + "step": 1250 + }, + { + "epoch": 0.13019218846869188, + "grad_norm": 1.0845853090286255, + "learning_rate": 0.0004820411704941223, + "loss": 1.3485, + "step": 1260 + }, + { + "epoch": 0.131225459805745, + "grad_norm": 1.1594990491867065, + "learning_rate": 0.00048173483914265976, + "loss": 1.4484, + "step": 1270 + }, + { + "epoch": 0.1322587311427981, + "grad_norm": 1.2621511220932007, + "learning_rate": 0.0004814260162446408, + "loss": 1.4147, + "step": 1280 + }, + { + "epoch": 0.1332920024798512, + "grad_norm": 1.6021978855133057, + "learning_rate": 0.00048111470512044065, + "loss": 1.3724, + "step": 1290 + }, + { + "epoch": 0.13432527381690432, + "grad_norm": 1.3557963371276855, + "learning_rate": 0.0004808009091171873, + "loss": 1.3844, + "step": 1300 + }, + { + "epoch": 0.13535854515395743, + "grad_norm": 1.341562271118164, + "learning_rate": 0.0004804846316087254, + "loss": 1.2895, + "step": 1310 + }, + { + "epoch": 0.13639181649101054, + "grad_norm": 1.1911447048187256, + "learning_rate": 0.00048016587599557996, + "loss": 1.3963, + "step": 1320 + }, + { + "epoch": 0.13742508782806365, + "grad_norm": 1.0620297193527222, + "learning_rate": 0.0004798446457049201, + "loss": 1.3621, + "step": 1330 + }, + { + "epoch": 0.13845835916511676, + "grad_norm": 1.1608030796051025, + "learning_rate": 0.0004795209441905217, + "loss": 1.3658, + "step": 1340 + }, + { + "epoch": 0.13949163050216987, + "grad_norm": 1.5855592489242554, + "learning_rate": 0.0004791947749327308, + "loss": 1.3683, + "step": 1350 + }, + { + "epoch": 0.14052490183922298, + "grad_norm": 1.5676512718200684, + "learning_rate": 0.0004788661414384255, + "loss": 1.3186, + "step": 1360 + }, + { + "epoch": 0.1415581731762761, + "grad_norm": 1.149056315422058, + "learning_rate": 0.0004785350472409792, + "loss": 1.3231, + "step": 1370 + }, + { + "epoch": 0.1425914445133292, + "grad_norm": 0.9153057336807251, + "learning_rate": 0.00047820149590022153, + "loss": 1.2976, + "step": 1380 + }, + { + "epoch": 0.1436247158503823, + "grad_norm": 1.2601430416107178, + "learning_rate": 0.00047786549100240107, + "loss": 1.4205, + "step": 1390 + }, + { + "epoch": 0.14465798718743542, + "grad_norm": 1.2112687826156616, + "learning_rate": 0.0004775270361601461, + "loss": 1.3192, + "step": 1400 + }, + { + "epoch": 0.14569125852448853, + "grad_norm": 1.1192986965179443, + "learning_rate": 0.00047718613501242604, + "loss": 1.355, + "step": 1410 + }, + { + "epoch": 0.14672452986154164, + "grad_norm": 1.2099722623825073, + "learning_rate": 0.00047684279122451223, + "loss": 1.3638, + "step": 1420 + }, + { + "epoch": 0.14775780119859475, + "grad_norm": 1.124934434890747, + "learning_rate": 0.00047649700848793886, + "loss": 1.331, + "step": 1430 + }, + { + "epoch": 0.14879107253564786, + "grad_norm": 1.2407630681991577, + "learning_rate": 0.00047614879052046254, + "loss": 1.5308, + "step": 1440 + }, + { + "epoch": 0.14982434387270097, + "grad_norm": 1.2991951704025269, + "learning_rate": 0.00047579814106602316, + "loss": 1.3365, + "step": 1450 + }, + { + "epoch": 0.15085761520975408, + "grad_norm": 1.3195995092391968, + "learning_rate": 0.0004754450638947032, + "loss": 1.3209, + "step": 1460 + }, + { + "epoch": 0.15189088654680719, + "grad_norm": 1.4838430881500244, + "learning_rate": 0.000475089562802687, + "loss": 1.4705, + "step": 1470 + }, + { + "epoch": 0.1529241578838603, + "grad_norm": 1.126265287399292, + "learning_rate": 0.0004747316416122205, + "loss": 1.3316, + "step": 1480 + }, + { + "epoch": 0.1539574292209134, + "grad_norm": 1.0109881162643433, + "learning_rate": 0.00047437130417156973, + "loss": 1.4045, + "step": 1490 + }, + { + "epoch": 0.15499070055796652, + "grad_norm": 0.8355922698974609, + "learning_rate": 0.00047400855435497945, + "loss": 1.3058, + "step": 1500 + }, + { + "epoch": 0.15602397189501963, + "grad_norm": 1.406782865524292, + "learning_rate": 0.00047364339606263185, + "loss": 1.2732, + "step": 1510 + }, + { + "epoch": 0.15705724323207274, + "grad_norm": 0.9180629849433899, + "learning_rate": 0.00047327583322060406, + "loss": 1.3472, + "step": 1520 + }, + { + "epoch": 0.15809051456912585, + "grad_norm": 1.1642565727233887, + "learning_rate": 0.00047290586978082654, + "loss": 1.4165, + "step": 1530 + }, + { + "epoch": 0.15912378590617896, + "grad_norm": 1.6167621612548828, + "learning_rate": 0.00047253350972104003, + "loss": 1.5038, + "step": 1540 + }, + { + "epoch": 0.16015705724323206, + "grad_norm": 1.1963645219802856, + "learning_rate": 0.00047215875704475314, + "loss": 1.3744, + "step": 1550 + }, + { + "epoch": 0.16119032858028517, + "grad_norm": 1.246496319770813, + "learning_rate": 0.00047178161578119925, + "loss": 1.3314, + "step": 1560 + }, + { + "epoch": 0.16222359991733828, + "grad_norm": 1.1050803661346436, + "learning_rate": 0.00047140208998529303, + "loss": 1.4149, + "step": 1570 + }, + { + "epoch": 0.1632568712543914, + "grad_norm": 1.379414439201355, + "learning_rate": 0.00047102018373758716, + "loss": 1.3817, + "step": 1580 + }, + { + "epoch": 0.1642901425914445, + "grad_norm": 1.0111711025238037, + "learning_rate": 0.00047063590114422804, + "loss": 1.2865, + "step": 1590 + }, + { + "epoch": 0.16532341392849761, + "grad_norm": 1.5682810544967651, + "learning_rate": 0.00047024924633691197, + "loss": 1.317, + "step": 1600 + }, + { + "epoch": 0.16635668526555072, + "grad_norm": 0.9005609154701233, + "learning_rate": 0.0004698602234728406, + "loss": 1.3659, + "step": 1610 + }, + { + "epoch": 0.16738995660260383, + "grad_norm": 1.00556218624115, + "learning_rate": 0.00046946883673467624, + "loss": 1.3373, + "step": 1620 + }, + { + "epoch": 0.16842322793965694, + "grad_norm": 1.387304425239563, + "learning_rate": 0.00046907509033049683, + "loss": 1.4858, + "step": 1630 + }, + { + "epoch": 0.16945649927671005, + "grad_norm": 1.2545139789581299, + "learning_rate": 0.0004686789884937509, + "loss": 1.4221, + "step": 1640 + }, + { + "epoch": 0.17048977061376316, + "grad_norm": 1.3092625141143799, + "learning_rate": 0.00046828053548321187, + "loss": 1.3609, + "step": 1650 + }, + { + "epoch": 0.17152304195081627, + "grad_norm": 1.0507373809814453, + "learning_rate": 0.0004678797355829322, + "loss": 1.3406, + "step": 1660 + }, + { + "epoch": 0.17255631328786938, + "grad_norm": 1.0271055698394775, + "learning_rate": 0.0004674765931021976, + "loss": 1.3098, + "step": 1670 + }, + { + "epoch": 0.1735895846249225, + "grad_norm": 1.1777933835983276, + "learning_rate": 0.0004670711123754804, + "loss": 1.2113, + "step": 1680 + }, + { + "epoch": 0.1746228559619756, + "grad_norm": 1.1888736486434937, + "learning_rate": 0.00046666329776239306, + "loss": 1.3137, + "step": 1690 + }, + { + "epoch": 0.1756561272990287, + "grad_norm": 1.4923595190048218, + "learning_rate": 0.00046625315364764156, + "loss": 1.23, + "step": 1700 + }, + { + "epoch": 0.17668939863608182, + "grad_norm": 1.3654290437698364, + "learning_rate": 0.00046584068444097766, + "loss": 1.3182, + "step": 1710 + }, + { + "epoch": 0.17772266997313493, + "grad_norm": 1.5486831665039062, + "learning_rate": 0.0004654258945771521, + "loss": 1.2854, + "step": 1720 + }, + { + "epoch": 0.17875594131018804, + "grad_norm": 1.3204468488693237, + "learning_rate": 0.0004650087885158667, + "loss": 1.3328, + "step": 1730 + }, + { + "epoch": 0.17978921264724115, + "grad_norm": 1.4773411750793457, + "learning_rate": 0.000464589370741726, + "loss": 1.3741, + "step": 1740 + }, + { + "epoch": 0.18082248398429426, + "grad_norm": 1.20607328414917, + "learning_rate": 0.00046416764576418993, + "loss": 1.26, + "step": 1750 + }, + { + "epoch": 0.18185575532134737, + "grad_norm": 1.2402129173278809, + "learning_rate": 0.00046374361811752436, + "loss": 1.2398, + "step": 1760 + }, + { + "epoch": 0.18288902665840048, + "grad_norm": 1.1439170837402344, + "learning_rate": 0.00046331729236075305, + "loss": 1.4529, + "step": 1770 + }, + { + "epoch": 0.1839222979954536, + "grad_norm": 1.268829107284546, + "learning_rate": 0.0004628886730776084, + "loss": 1.4535, + "step": 1780 + }, + { + "epoch": 0.1849555693325067, + "grad_norm": 1.37249755859375, + "learning_rate": 0.0004624577648764819, + "loss": 1.3249, + "step": 1790 + }, + { + "epoch": 0.1859888406695598, + "grad_norm": 1.2565507888793945, + "learning_rate": 0.0004620245723903751, + "loss": 1.2866, + "step": 1800 + }, + { + "epoch": 0.18702211200661292, + "grad_norm": 1.1188476085662842, + "learning_rate": 0.0004615891002768494, + "loss": 1.1712, + "step": 1810 + }, + { + "epoch": 0.18805538334366606, + "grad_norm": 1.3613345623016357, + "learning_rate": 0.00046115135321797617, + "loss": 1.1788, + "step": 1820 + }, + { + "epoch": 0.18908865468071917, + "grad_norm": 1.3465733528137207, + "learning_rate": 0.00046071133592028626, + "loss": 1.2391, + "step": 1830 + }, + { + "epoch": 0.19012192601777228, + "grad_norm": 1.2718288898468018, + "learning_rate": 0.00046026905311471946, + "loss": 1.2213, + "step": 1840 + }, + { + "epoch": 0.1911551973548254, + "grad_norm": 1.372878909111023, + "learning_rate": 0.0004598245095565738, + "loss": 1.2036, + "step": 1850 + }, + { + "epoch": 0.1921884686918785, + "grad_norm": 1.198723316192627, + "learning_rate": 0.00045937771002545403, + "loss": 1.192, + "step": 1860 + }, + { + "epoch": 0.1932217400289316, + "grad_norm": 0.9199055433273315, + "learning_rate": 0.00045892865932522077, + "loss": 1.2999, + "step": 1870 + }, + { + "epoch": 0.19425501136598472, + "grad_norm": 1.3330020904541016, + "learning_rate": 0.0004584773622839383, + "loss": 1.3587, + "step": 1880 + }, + { + "epoch": 0.19528828270303783, + "grad_norm": 1.228247046470642, + "learning_rate": 0.0004580238237538232, + "loss": 1.4654, + "step": 1890 + }, + { + "epoch": 0.19632155404009094, + "grad_norm": 1.6616053581237793, + "learning_rate": 0.0004575680486111915, + "loss": 1.1583, + "step": 1900 + }, + { + "epoch": 0.19735482537714405, + "grad_norm": 1.0149476528167725, + "learning_rate": 0.000457110041756407, + "loss": 1.2121, + "step": 1910 + }, + { + "epoch": 0.19838809671419716, + "grad_norm": 1.6001296043395996, + "learning_rate": 0.00045664980811382813, + "loss": 1.2938, + "step": 1920 + }, + { + "epoch": 0.19942136805125027, + "grad_norm": 1.263951301574707, + "learning_rate": 0.00045618735263175495, + "loss": 1.1885, + "step": 1930 + }, + { + "epoch": 0.20045463938830338, + "grad_norm": 1.4672034978866577, + "learning_rate": 0.00045572268028237637, + "loss": 1.2981, + "step": 1940 + }, + { + "epoch": 0.2014879107253565, + "grad_norm": 1.1751412153244019, + "learning_rate": 0.0004552557960617161, + "loss": 1.117, + "step": 1950 + }, + { + "epoch": 0.2025211820624096, + "grad_norm": 1.131339192390442, + "learning_rate": 0.0004547867049895796, + "loss": 1.1909, + "step": 1960 + }, + { + "epoch": 0.2035544533994627, + "grad_norm": 0.991729736328125, + "learning_rate": 0.0004543154121094996, + "loss": 1.1328, + "step": 1970 + }, + { + "epoch": 0.20458772473651582, + "grad_norm": 0.9073534607887268, + "learning_rate": 0.0004538419224886819, + "loss": 0.9973, + "step": 1980 + }, + { + "epoch": 0.20562099607356893, + "grad_norm": 1.2480231523513794, + "learning_rate": 0.00045336624121795124, + "loss": 1.2367, + "step": 1990 + }, + { + "epoch": 0.20665426741062204, + "grad_norm": 1.28248131275177, + "learning_rate": 0.0004528883734116963, + "loss": 1.1738, + "step": 2000 + }, + { + "epoch": 0.20768753874767515, + "grad_norm": 1.386885166168213, + "learning_rate": 0.00045240832420781474, + "loss": 1.3788, + "step": 2010 + }, + { + "epoch": 0.20872081008472826, + "grad_norm": 1.1158126592636108, + "learning_rate": 0.0004519260987676579, + "loss": 1.3046, + "step": 2020 + }, + { + "epoch": 0.20975408142178137, + "grad_norm": 1.162636399269104, + "learning_rate": 0.00045144170227597545, + "loss": 1.2355, + "step": 2030 + }, + { + "epoch": 0.21078735275883448, + "grad_norm": 1.4077121019363403, + "learning_rate": 0.0004509551399408598, + "loss": 1.1542, + "step": 2040 + }, + { + "epoch": 0.2118206240958876, + "grad_norm": 1.1237014532089233, + "learning_rate": 0.00045046641699368953, + "loss": 1.3411, + "step": 2050 + }, + { + "epoch": 0.2128538954329407, + "grad_norm": 1.1503592729568481, + "learning_rate": 0.0004499755386890736, + "loss": 1.1783, + "step": 2060 + }, + { + "epoch": 0.2138871667699938, + "grad_norm": 1.2852717638015747, + "learning_rate": 0.0004494825103047949, + "loss": 1.3331, + "step": 2070 + }, + { + "epoch": 0.21492043810704692, + "grad_norm": 1.2611557245254517, + "learning_rate": 0.0004489873371417534, + "loss": 1.3272, + "step": 2080 + }, + { + "epoch": 0.21595370944410003, + "grad_norm": 1.4016764163970947, + "learning_rate": 0.00044849002452390874, + "loss": 1.1658, + "step": 2090 + }, + { + "epoch": 0.21698698078115314, + "grad_norm": 1.4920654296875, + "learning_rate": 0.0004479905777982238, + "loss": 1.2651, + "step": 2100 + }, + { + "epoch": 0.21802025211820625, + "grad_norm": 1.1262540817260742, + "learning_rate": 0.0004474890023346066, + "loss": 1.2347, + "step": 2110 + }, + { + "epoch": 0.21905352345525936, + "grad_norm": 1.2998685836791992, + "learning_rate": 0.0004469853035258526, + "loss": 1.3167, + "step": 2120 + }, + { + "epoch": 0.22008679479231247, + "grad_norm": 1.0999696254730225, + "learning_rate": 0.0004464794867875871, + "loss": 1.1621, + "step": 2130 + }, + { + "epoch": 0.22112006612936558, + "grad_norm": 0.9801570177078247, + "learning_rate": 0.0004459715575582066, + "loss": 1.1631, + "step": 2140 + }, + { + "epoch": 0.2221533374664187, + "grad_norm": 1.161098837852478, + "learning_rate": 0.00044546152129882054, + "loss": 1.1256, + "step": 2150 + }, + { + "epoch": 0.2231866088034718, + "grad_norm": 1.2714890241622925, + "learning_rate": 0.0004449493834931927, + "loss": 1.1182, + "step": 2160 + }, + { + "epoch": 0.2242198801405249, + "grad_norm": 0.9561129808425903, + "learning_rate": 0.0004444351496476818, + "loss": 1.2051, + "step": 2170 + }, + { + "epoch": 0.22525315147757802, + "grad_norm": 1.3160920143127441, + "learning_rate": 0.0004439188252911828, + "loss": 1.2268, + "step": 2180 + }, + { + "epoch": 0.22628642281463113, + "grad_norm": 1.1478712558746338, + "learning_rate": 0.0004434004159750671, + "loss": 1.1928, + "step": 2190 + }, + { + "epoch": 0.22731969415168424, + "grad_norm": 1.2426934242248535, + "learning_rate": 0.0004428799272731231, + "loss": 1.1548, + "step": 2200 + }, + { + "epoch": 0.22835296548873735, + "grad_norm": 1.3587703704833984, + "learning_rate": 0.0004423573647814961, + "loss": 1.1371, + "step": 2210 + }, + { + "epoch": 0.22938623682579046, + "grad_norm": 1.0638786554336548, + "learning_rate": 0.0004418327341186282, + "loss": 1.1668, + "step": 2220 + }, + { + "epoch": 0.23041950816284357, + "grad_norm": 1.1600587368011475, + "learning_rate": 0.00044130604092519794, + "loss": 1.1524, + "step": 2230 + }, + { + "epoch": 0.23145277949989668, + "grad_norm": 1.402921199798584, + "learning_rate": 0.0004407772908640595, + "loss": 1.3963, + "step": 2240 + }, + { + "epoch": 0.2324860508369498, + "grad_norm": 1.3043389320373535, + "learning_rate": 0.00044024648962018184, + "loss": 1.2359, + "step": 2250 + }, + { + "epoch": 0.2335193221740029, + "grad_norm": 1.1807844638824463, + "learning_rate": 0.0004397136429005879, + "loss": 1.366, + "step": 2260 + }, + { + "epoch": 0.234552593511056, + "grad_norm": 1.397064208984375, + "learning_rate": 0.00043917875643429284, + "loss": 1.234, + "step": 2270 + }, + { + "epoch": 0.23558586484810912, + "grad_norm": 1.3520923852920532, + "learning_rate": 0.0004386418359722426, + "loss": 1.1863, + "step": 2280 + }, + { + "epoch": 0.23661913618516223, + "grad_norm": 1.3684437274932861, + "learning_rate": 0.00043810288728725203, + "loss": 1.3008, + "step": 2290 + }, + { + "epoch": 0.23765240752221534, + "grad_norm": 1.1893550157546997, + "learning_rate": 0.0004375619161739428, + "loss": 1.0295, + "step": 2300 + }, + { + "epoch": 0.23868567885926845, + "grad_norm": 1.57694673538208, + "learning_rate": 0.0004370189284486814, + "loss": 1.2778, + "step": 2310 + }, + { + "epoch": 0.23971895019632156, + "grad_norm": 1.601629376411438, + "learning_rate": 0.0004364739299495162, + "loss": 1.0583, + "step": 2320 + }, + { + "epoch": 0.24075222153337467, + "grad_norm": 1.3246372938156128, + "learning_rate": 0.00043592692653611465, + "loss": 1.201, + "step": 2330 + }, + { + "epoch": 0.24178549287042778, + "grad_norm": 1.020665168762207, + "learning_rate": 0.0004353779240897008, + "loss": 1.3127, + "step": 2340 + }, + { + "epoch": 0.24281876420748089, + "grad_norm": 1.082587480545044, + "learning_rate": 0.00043482692851299146, + "loss": 1.225, + "step": 2350 + }, + { + "epoch": 0.243852035544534, + "grad_norm": 1.093636155128479, + "learning_rate": 0.00043427394573013314, + "loss": 1.0587, + "step": 2360 + }, + { + "epoch": 0.2448853068815871, + "grad_norm": 1.3400568962097168, + "learning_rate": 0.00043371898168663816, + "loss": 1.2375, + "step": 2370 + }, + { + "epoch": 0.24591857821864022, + "grad_norm": 1.223877191543579, + "learning_rate": 0.00043316204234932084, + "loss": 1.3076, + "step": 2380 + }, + { + "epoch": 0.24695184955569333, + "grad_norm": 1.4696872234344482, + "learning_rate": 0.0004326031337062333, + "loss": 1.251, + "step": 2390 + }, + { + "epoch": 0.24798512089274644, + "grad_norm": 1.3883171081542969, + "learning_rate": 0.00043204226176660107, + "loss": 1.2215, + "step": 2400 + }, + { + "epoch": 0.24901839222979955, + "grad_norm": 1.1045054197311401, + "learning_rate": 0.0004314794325607584, + "loss": 1.1014, + "step": 2410 + }, + { + "epoch": 0.25005166356685266, + "grad_norm": 1.273431658744812, + "learning_rate": 0.00043091465214008354, + "loss": 1.0845, + "step": 2420 + }, + { + "epoch": 0.25108493490390577, + "grad_norm": 1.0618531703948975, + "learning_rate": 0.0004303479265769337, + "loss": 1.1162, + "step": 2430 + }, + { + "epoch": 0.2521182062409589, + "grad_norm": 1.3230130672454834, + "learning_rate": 0.0004297792619645797, + "loss": 1.0385, + "step": 2440 + }, + { + "epoch": 0.253151477578012, + "grad_norm": 1.3020095825195312, + "learning_rate": 0.0004292086644171403, + "loss": 1.1548, + "step": 2450 + }, + { + "epoch": 0.2541847489150651, + "grad_norm": 1.3359605073928833, + "learning_rate": 0.0004286361400695169, + "loss": 1.3534, + "step": 2460 + }, + { + "epoch": 0.2552180202521182, + "grad_norm": 0.9598666429519653, + "learning_rate": 0.00042806169507732706, + "loss": 1.2432, + "step": 2470 + }, + { + "epoch": 0.2562512915891713, + "grad_norm": 0.9668743014335632, + "learning_rate": 0.00042748533561683865, + "loss": 1.2512, + "step": 2480 + }, + { + "epoch": 0.2572845629262244, + "grad_norm": 1.0706520080566406, + "learning_rate": 0.0004269070678849034, + "loss": 1.136, + "step": 2490 + }, + { + "epoch": 0.25831783426327753, + "grad_norm": 1.0470647811889648, + "learning_rate": 0.00042632689809889027, + "loss": 1.1211, + "step": 2500 + }, + { + "epoch": 0.25935110560033064, + "grad_norm": 1.1314566135406494, + "learning_rate": 0.0004257448324966183, + "loss": 1.0704, + "step": 2510 + }, + { + "epoch": 0.26038437693738375, + "grad_norm": 1.085749626159668, + "learning_rate": 0.00042516087733629004, + "loss": 1.1727, + "step": 2520 + }, + { + "epoch": 0.26141764827443686, + "grad_norm": 1.2759054899215698, + "learning_rate": 0.00042457503889642396, + "loss": 1.2864, + "step": 2530 + }, + { + "epoch": 0.26245091961149, + "grad_norm": 1.2468546628952026, + "learning_rate": 0.0004239873234757871, + "loss": 1.0734, + "step": 2540 + }, + { + "epoch": 0.2634841909485431, + "grad_norm": 1.2701605558395386, + "learning_rate": 0.00042339773739332706, + "loss": 1.1816, + "step": 2550 + }, + { + "epoch": 0.2645174622855962, + "grad_norm": 1.0027897357940674, + "learning_rate": 0.00042280628698810447, + "loss": 1.0726, + "step": 2560 + }, + { + "epoch": 0.2655507336226493, + "grad_norm": 1.4462809562683105, + "learning_rate": 0.0004222129786192245, + "loss": 1.278, + "step": 2570 + }, + { + "epoch": 0.2665840049597024, + "grad_norm": 1.241769552230835, + "learning_rate": 0.0004216178186657686, + "loss": 1.1829, + "step": 2580 + }, + { + "epoch": 0.2676172762967555, + "grad_norm": 1.6216635704040527, + "learning_rate": 0.00042102081352672594, + "loss": 1.1479, + "step": 2590 + }, + { + "epoch": 0.26865054763380863, + "grad_norm": 4.33831787109375, + "learning_rate": 0.0004204219696209248, + "loss": 1.1709, + "step": 2600 + }, + { + "epoch": 0.26968381897086174, + "grad_norm": 0.952393114566803, + "learning_rate": 0.00041982129338696296, + "loss": 1.1018, + "step": 2610 + }, + { + "epoch": 0.27071709030791485, + "grad_norm": 1.4484752416610718, + "learning_rate": 0.0004192187912831393, + "loss": 1.081, + "step": 2620 + }, + { + "epoch": 0.27175036164496796, + "grad_norm": 1.0843290090560913, + "learning_rate": 0.0004186144697873835, + "loss": 1.251, + "step": 2630 + }, + { + "epoch": 0.2727836329820211, + "grad_norm": 1.5165774822235107, + "learning_rate": 0.0004180083353971871, + "loss": 1.134, + "step": 2640 + }, + { + "epoch": 0.2738169043190742, + "grad_norm": 1.2171909809112549, + "learning_rate": 0.00041740039462953343, + "loss": 1.1763, + "step": 2650 + }, + { + "epoch": 0.2748501756561273, + "grad_norm": 1.0561660528182983, + "learning_rate": 0.00041679065402082724, + "loss": 1.1178, + "step": 2660 + }, + { + "epoch": 0.2758834469931804, + "grad_norm": 1.2738356590270996, + "learning_rate": 0.0004161791201268247, + "loss": 1.2379, + "step": 2670 + }, + { + "epoch": 0.2769167183302335, + "grad_norm": 1.127266764640808, + "learning_rate": 0.000415565799522563, + "loss": 1.2347, + "step": 2680 + }, + { + "epoch": 0.2779499896672866, + "grad_norm": 1.4078705310821533, + "learning_rate": 0.0004149506988022894, + "loss": 1.2155, + "step": 2690 + }, + { + "epoch": 0.27898326100433973, + "grad_norm": 0.9871230125427246, + "learning_rate": 0.0004143338245793906, + "loss": 1.1583, + "step": 2700 + }, + { + "epoch": 0.28001653234139284, + "grad_norm": 0.8731614947319031, + "learning_rate": 0.0004137151834863213, + "loss": 1.171, + "step": 2710 + }, + { + "epoch": 0.28104980367844595, + "grad_norm": 1.0814101696014404, + "learning_rate": 0.0004130947821745332, + "loss": 1.2112, + "step": 2720 + }, + { + "epoch": 0.28208307501549906, + "grad_norm": 1.695854902267456, + "learning_rate": 0.0004124726273144034, + "loss": 1.0269, + "step": 2730 + }, + { + "epoch": 0.2831163463525522, + "grad_norm": 0.9004189372062683, + "learning_rate": 0.00041184872559516244, + "loss": 1.1974, + "step": 2740 + }, + { + "epoch": 0.2841496176896053, + "grad_norm": 1.034590482711792, + "learning_rate": 0.0004112230837248229, + "loss": 1.1068, + "step": 2750 + }, + { + "epoch": 0.2851828890266584, + "grad_norm": 1.4124444723129272, + "learning_rate": 0.0004105957084301069, + "loss": 1.0922, + "step": 2760 + }, + { + "epoch": 0.2862161603637115, + "grad_norm": 1.4991061687469482, + "learning_rate": 0.00040996660645637345, + "loss": 1.2392, + "step": 2770 + }, + { + "epoch": 0.2872494317007646, + "grad_norm": 1.316277027130127, + "learning_rate": 0.00040933578456754687, + "loss": 1.0433, + "step": 2780 + }, + { + "epoch": 0.2882827030378177, + "grad_norm": 1.6624524593353271, + "learning_rate": 0.00040870324954604323, + "loss": 0.9623, + "step": 2790 + }, + { + "epoch": 0.28931597437487083, + "grad_norm": 1.2658953666687012, + "learning_rate": 0.00040806900819269766, + "loss": 1.108, + "step": 2800 + }, + { + "epoch": 0.29034924571192394, + "grad_norm": 1.3295769691467285, + "learning_rate": 0.00040743306732669133, + "loss": 1.137, + "step": 2810 + }, + { + "epoch": 0.29138251704897705, + "grad_norm": 1.6526551246643066, + "learning_rate": 0.0004067954337854783, + "loss": 1.1281, + "step": 2820 + }, + { + "epoch": 0.29241578838603016, + "grad_norm": 1.1587738990783691, + "learning_rate": 0.0004061561144247115, + "loss": 1.0086, + "step": 2830 + }, + { + "epoch": 0.29344905972308327, + "grad_norm": 1.454256534576416, + "learning_rate": 0.0004055151161181694, + "loss": 1.0723, + "step": 2840 + }, + { + "epoch": 0.2944823310601364, + "grad_norm": 1.476521611213684, + "learning_rate": 0.00040487244575768193, + "loss": 1.2486, + "step": 2850 + }, + { + "epoch": 0.2955156023971895, + "grad_norm": 0.9193058609962463, + "learning_rate": 0.00040422811025305676, + "loss": 1.1071, + "step": 2860 + }, + { + "epoch": 0.2965488737342426, + "grad_norm": 1.8984912633895874, + "learning_rate": 0.0004035821165320045, + "loss": 1.1724, + "step": 2870 + }, + { + "epoch": 0.2975821450712957, + "grad_norm": 1.7232333421707153, + "learning_rate": 0.00040293447154006435, + "loss": 1.174, + "step": 2880 + }, + { + "epoch": 0.2986154164083488, + "grad_norm": 0.9074342846870422, + "learning_rate": 0.0004022851822405297, + "loss": 1.2448, + "step": 2890 + }, + { + "epoch": 0.29964868774540193, + "grad_norm": 1.2896414995193481, + "learning_rate": 0.0004016342556143727, + "loss": 1.1207, + "step": 2900 + }, + { + "epoch": 0.30068195908245504, + "grad_norm": 1.264952301979065, + "learning_rate": 0.0004009816986601702, + "loss": 1.0868, + "step": 2910 + }, + { + "epoch": 0.30171523041950815, + "grad_norm": 1.1345736980438232, + "learning_rate": 0.00040032751839402727, + "loss": 1.1881, + "step": 2920 + }, + { + "epoch": 0.30274850175656126, + "grad_norm": 1.0581121444702148, + "learning_rate": 0.00039967172184950263, + "loss": 1.1796, + "step": 2930 + }, + { + "epoch": 0.30378177309361437, + "grad_norm": 1.1877319812774658, + "learning_rate": 0.0003990143160775327, + "loss": 1.108, + "step": 2940 + }, + { + "epoch": 0.3048150444306675, + "grad_norm": 1.0478935241699219, + "learning_rate": 0.0003983553081463559, + "loss": 0.9986, + "step": 2950 + }, + { + "epoch": 0.3058483157677206, + "grad_norm": 1.1299808025360107, + "learning_rate": 0.0003976947051414366, + "loss": 1.0221, + "step": 2960 + }, + { + "epoch": 0.3068815871047737, + "grad_norm": 1.0618709325790405, + "learning_rate": 0.00039703251416538883, + "loss": 0.9024, + "step": 2970 + }, + { + "epoch": 0.3079148584418268, + "grad_norm": 1.099435806274414, + "learning_rate": 0.0003963687423379003, + "loss": 1.2754, + "step": 2980 + }, + { + "epoch": 0.3089481297788799, + "grad_norm": 1.1851657629013062, + "learning_rate": 0.00039570339679565527, + "loss": 1.1154, + "step": 2990 + }, + { + "epoch": 0.30998140111593303, + "grad_norm": 1.1812468767166138, + "learning_rate": 0.00039503648469225826, + "loss": 0.9979, + "step": 3000 + }, + { + "epoch": 0.31101467245298614, + "grad_norm": 1.1941215991973877, + "learning_rate": 0.00039436801319815717, + "loss": 1.0625, + "step": 3010 + }, + { + "epoch": 0.31204794379003925, + "grad_norm": 0.9969115853309631, + "learning_rate": 0.00039369798950056567, + "loss": 1.189, + "step": 3020 + }, + { + "epoch": 0.31308121512709236, + "grad_norm": 1.1364282369613647, + "learning_rate": 0.00039302642080338664, + "loss": 1.107, + "step": 3030 + }, + { + "epoch": 0.31411448646414547, + "grad_norm": 1.234078288078308, + "learning_rate": 0.0003923533143271341, + "loss": 1.238, + "step": 3040 + }, + { + "epoch": 0.3151477578011986, + "grad_norm": 1.0862330198287964, + "learning_rate": 0.0003916786773088559, + "loss": 1.1151, + "step": 3050 + }, + { + "epoch": 0.3161810291382517, + "grad_norm": 0.7923426032066345, + "learning_rate": 0.0003910025170020558, + "loss": 1.1031, + "step": 3060 + }, + { + "epoch": 0.3172143004753048, + "grad_norm": 1.6818230152130127, + "learning_rate": 0.0003903248406766158, + "loss": 1.0546, + "step": 3070 + }, + { + "epoch": 0.3182475718123579, + "grad_norm": 1.0749332904815674, + "learning_rate": 0.00038964565561871723, + "loss": 1.1412, + "step": 3080 + }, + { + "epoch": 0.319280843149411, + "grad_norm": 1.6896083354949951, + "learning_rate": 0.00038896496913076327, + "loss": 1.0668, + "step": 3090 + }, + { + "epoch": 0.32031411448646413, + "grad_norm": 1.24112069606781, + "learning_rate": 0.0003882827885312998, + "loss": 1.2632, + "step": 3100 + }, + { + "epoch": 0.32134738582351724, + "grad_norm": 1.2017937898635864, + "learning_rate": 0.00038759912115493724, + "loss": 1.0969, + "step": 3110 + }, + { + "epoch": 0.32238065716057035, + "grad_norm": 1.20210599899292, + "learning_rate": 0.00038691397435227097, + "loss": 0.9202, + "step": 3120 + }, + { + "epoch": 0.32341392849762346, + "grad_norm": 1.408257246017456, + "learning_rate": 0.0003862273554898032, + "loss": 1.1583, + "step": 3130 + }, + { + "epoch": 0.32444719983467657, + "grad_norm": 1.5274341106414795, + "learning_rate": 0.000385539271949863, + "loss": 1.0382, + "step": 3140 + }, + { + "epoch": 0.3254804711717297, + "grad_norm": 1.1086093187332153, + "learning_rate": 0.00038484973113052736, + "loss": 1.1232, + "step": 3150 + }, + { + "epoch": 0.3265137425087828, + "grad_norm": 1.35590660572052, + "learning_rate": 0.0003841587404455413, + "loss": 1.1019, + "step": 3160 + }, + { + "epoch": 0.3275470138458359, + "grad_norm": 1.0598667860031128, + "learning_rate": 0.000383466307324239, + "loss": 1.1557, + "step": 3170 + }, + { + "epoch": 0.328580285182889, + "grad_norm": 1.1116646528244019, + "learning_rate": 0.00038277243921146253, + "loss": 1.0634, + "step": 3180 + }, + { + "epoch": 0.3296135565199421, + "grad_norm": 1.4738123416900635, + "learning_rate": 0.0003820771435674829, + "loss": 1.0067, + "step": 3190 + }, + { + "epoch": 0.33064682785699523, + "grad_norm": 1.3569648265838623, + "learning_rate": 0.00038138042786791987, + "loss": 1.0798, + "step": 3200 + }, + { + "epoch": 0.33168009919404834, + "grad_norm": 1.073232650756836, + "learning_rate": 0.00038068229960366054, + "loss": 1.0349, + "step": 3210 + }, + { + "epoch": 0.33271337053110145, + "grad_norm": 1.4980833530426025, + "learning_rate": 0.0003799827662807801, + "loss": 1.1281, + "step": 3220 + }, + { + "epoch": 0.33374664186815456, + "grad_norm": 1.4308544397354126, + "learning_rate": 0.0003792818354204601, + "loss": 1.1056, + "step": 3230 + }, + { + "epoch": 0.33477991320520767, + "grad_norm": 1.5736769437789917, + "learning_rate": 0.0003785795145589085, + "loss": 1.0928, + "step": 3240 + }, + { + "epoch": 0.3358131845422608, + "grad_norm": 1.5231635570526123, + "learning_rate": 0.0003778758112472776, + "loss": 1.2857, + "step": 3250 + }, + { + "epoch": 0.3368464558793139, + "grad_norm": 1.0374634265899658, + "learning_rate": 0.00037717073305158376, + "loss": 1.1738, + "step": 3260 + }, + { + "epoch": 0.337879727216367, + "grad_norm": 1.1238751411437988, + "learning_rate": 0.0003764642875526256, + "loss": 0.9849, + "step": 3270 + }, + { + "epoch": 0.3389129985534201, + "grad_norm": 1.039632797241211, + "learning_rate": 0.0003757564823459025, + "loss": 1.059, + "step": 3280 + }, + { + "epoch": 0.3399462698904732, + "grad_norm": 1.1179864406585693, + "learning_rate": 0.0003750473250415334, + "loss": 0.9745, + "step": 3290 + }, + { + "epoch": 0.34097954122752633, + "grad_norm": 1.1467711925506592, + "learning_rate": 0.0003743368232641741, + "loss": 1.1077, + "step": 3300 + }, + { + "epoch": 0.34201281256457944, + "grad_norm": 1.0984638929367065, + "learning_rate": 0.00037362498465293604, + "loss": 1.0454, + "step": 3310 + }, + { + "epoch": 0.34304608390163255, + "grad_norm": 1.0687668323516846, + "learning_rate": 0.000372911816861304, + "loss": 1.073, + "step": 3320 + }, + { + "epoch": 0.34407935523868566, + "grad_norm": 1.6367201805114746, + "learning_rate": 0.00037219732755705353, + "loss": 1.1094, + "step": 3330 + }, + { + "epoch": 0.34511262657573877, + "grad_norm": 1.2905350923538208, + "learning_rate": 0.0003714815244221689, + "loss": 1.1536, + "step": 3340 + }, + { + "epoch": 0.3461458979127919, + "grad_norm": 0.9039587378501892, + "learning_rate": 0.00037076441515276003, + "loss": 1.1906, + "step": 3350 + }, + { + "epoch": 0.347179169249845, + "grad_norm": 1.3063287734985352, + "learning_rate": 0.0003700460074589804, + "loss": 1.1448, + "step": 3360 + }, + { + "epoch": 0.3482124405868981, + "grad_norm": 1.5665302276611328, + "learning_rate": 0.00036932630906494346, + "loss": 1.2537, + "step": 3370 + }, + { + "epoch": 0.3492457119239512, + "grad_norm": 1.0483524799346924, + "learning_rate": 0.0003686053277086401, + "loss": 1.1135, + "step": 3380 + }, + { + "epoch": 0.3502789832610043, + "grad_norm": 1.2478828430175781, + "learning_rate": 0.0003678830711418551, + "loss": 1.0816, + "step": 3390 + }, + { + "epoch": 0.3513122545980574, + "grad_norm": 1.14175283908844, + "learning_rate": 0.00036715954713008406, + "loss": 1.1217, + "step": 3400 + }, + { + "epoch": 0.35234552593511054, + "grad_norm": 1.0000718832015991, + "learning_rate": 0.0003664347634524497, + "loss": 1.1127, + "step": 3410 + }, + { + "epoch": 0.35337879727216365, + "grad_norm": 1.5337084531784058, + "learning_rate": 0.00036570872790161834, + "loss": 1.0619, + "step": 3420 + }, + { + "epoch": 0.35441206860921676, + "grad_norm": 0.878587543964386, + "learning_rate": 0.00036498144828371604, + "loss": 1.1094, + "step": 3430 + }, + { + "epoch": 0.35544533994626987, + "grad_norm": 1.3913145065307617, + "learning_rate": 0.0003642529324182449, + "loss": 0.9874, + "step": 3440 + }, + { + "epoch": 0.356478611283323, + "grad_norm": 1.388934850692749, + "learning_rate": 0.0003635231881379985, + "loss": 1.0177, + "step": 3450 + }, + { + "epoch": 0.3575118826203761, + "grad_norm": 1.2259199619293213, + "learning_rate": 0.00036279222328897826, + "loss": 1.0533, + "step": 3460 + }, + { + "epoch": 0.3585451539574292, + "grad_norm": 1.561023235321045, + "learning_rate": 0.0003620600457303085, + "loss": 1.1231, + "step": 3470 + }, + { + "epoch": 0.3595784252944823, + "grad_norm": 1.6612300872802734, + "learning_rate": 0.0003613266633341528, + "loss": 1.0971, + "step": 3480 + }, + { + "epoch": 0.3606116966315354, + "grad_norm": 1.3272353410720825, + "learning_rate": 0.00036059208398562793, + "loss": 1.024, + "step": 3490 + }, + { + "epoch": 0.3616449679685885, + "grad_norm": 1.2441381216049194, + "learning_rate": 0.00035985631558272074, + "loss": 1.0369, + "step": 3500 + }, + { + "epoch": 0.36267823930564164, + "grad_norm": 1.1805377006530762, + "learning_rate": 0.00035911936603620225, + "loss": 0.9626, + "step": 3510 + }, + { + "epoch": 0.36371151064269475, + "grad_norm": 1.3905142545700073, + "learning_rate": 0.00035838124326954254, + "loss": 1.1415, + "step": 3520 + }, + { + "epoch": 0.36474478197974786, + "grad_norm": 1.2646347284317017, + "learning_rate": 0.0003576419552188261, + "loss": 1.0101, + "step": 3530 + }, + { + "epoch": 0.36577805331680097, + "grad_norm": 1.1667832136154175, + "learning_rate": 0.000356901509832666, + "loss": 0.9862, + "step": 3540 + }, + { + "epoch": 0.3668113246538541, + "grad_norm": 0.8571506142616272, + "learning_rate": 0.00035615991507211887, + "loss": 0.9644, + "step": 3550 + }, + { + "epoch": 0.3678445959909072, + "grad_norm": 0.9799628853797913, + "learning_rate": 0.00035541717891059886, + "loss": 0.8918, + "step": 3560 + }, + { + "epoch": 0.3688778673279603, + "grad_norm": 1.0068583488464355, + "learning_rate": 0.0003546733093337924, + "loss": 0.935, + "step": 3570 + }, + { + "epoch": 0.3699111386650134, + "grad_norm": 0.8947041034698486, + "learning_rate": 0.0003539283143395719, + "loss": 1.0752, + "step": 3580 + }, + { + "epoch": 0.3709444100020665, + "grad_norm": 1.0797079801559448, + "learning_rate": 0.00035318220193790976, + "loss": 0.9534, + "step": 3590 + }, + { + "epoch": 0.3719776813391196, + "grad_norm": 1.0746815204620361, + "learning_rate": 0.0003524349801507929, + "loss": 0.9862, + "step": 3600 + }, + { + "epoch": 0.37301095267617274, + "grad_norm": 1.0419459342956543, + "learning_rate": 0.0003516866570121356, + "loss": 1.0606, + "step": 3610 + }, + { + "epoch": 0.37404422401322585, + "grad_norm": 1.1848483085632324, + "learning_rate": 0.0003509372405676937, + "loss": 0.97, + "step": 3620 + }, + { + "epoch": 0.375077495350279, + "grad_norm": 1.2240206003189087, + "learning_rate": 0.00035018673887497807, + "loss": 1.0237, + "step": 3630 + }, + { + "epoch": 0.3761107666873321, + "grad_norm": 1.6243517398834229, + "learning_rate": 0.0003494351600031677, + "loss": 1.0748, + "step": 3640 + }, + { + "epoch": 0.37714403802438523, + "grad_norm": 1.4265719652175903, + "learning_rate": 0.00034868251203302314, + "loss": 1.0639, + "step": 3650 + }, + { + "epoch": 0.37817730936143834, + "grad_norm": 1.0866336822509766, + "learning_rate": 0.00034792880305679967, + "loss": 0.9635, + "step": 3660 + }, + { + "epoch": 0.37921058069849145, + "grad_norm": 1.0087511539459229, + "learning_rate": 0.00034717404117816, + "loss": 1.1082, + "step": 3670 + }, + { + "epoch": 0.38024385203554456, + "grad_norm": 0.9332922697067261, + "learning_rate": 0.0003464182345120877, + "loss": 1.0258, + "step": 3680 + }, + { + "epoch": 0.38127712337259767, + "grad_norm": 0.9739323854446411, + "learning_rate": 0.000345661391184799, + "loss": 1.0168, + "step": 3690 + }, + { + "epoch": 0.3823103947096508, + "grad_norm": 0.912225067615509, + "learning_rate": 0.00034490351933365673, + "loss": 1.0996, + "step": 3700 + }, + { + "epoch": 0.3833436660467039, + "grad_norm": 0.853449285030365, + "learning_rate": 0.0003441446271070816, + "loss": 0.9759, + "step": 3710 + }, + { + "epoch": 0.384376937383757, + "grad_norm": 1.4455146789550781, + "learning_rate": 0.0003433847226644653, + "loss": 0.9917, + "step": 3720 + }, + { + "epoch": 0.3854102087208101, + "grad_norm": 0.9426379799842834, + "learning_rate": 0.0003426238141760826, + "loss": 1.1549, + "step": 3730 + }, + { + "epoch": 0.3864434800578632, + "grad_norm": 1.3207985162734985, + "learning_rate": 0.00034186190982300323, + "loss": 1.0742, + "step": 3740 + }, + { + "epoch": 0.38747675139491633, + "grad_norm": 0.9599164128303528, + "learning_rate": 0.00034109901779700474, + "loss": 1.0011, + "step": 3750 + }, + { + "epoch": 0.38851002273196944, + "grad_norm": 1.0271940231323242, + "learning_rate": 0.0003403351463004831, + "loss": 0.9508, + "step": 3760 + }, + { + "epoch": 0.38954329406902255, + "grad_norm": 1.680550456047058, + "learning_rate": 0.0003395703035463659, + "loss": 1.134, + "step": 3770 + }, + { + "epoch": 0.39057656540607566, + "grad_norm": 1.4388446807861328, + "learning_rate": 0.00033880449775802306, + "loss": 1.1811, + "step": 3780 + }, + { + "epoch": 0.39160983674312877, + "grad_norm": 1.1560338735580444, + "learning_rate": 0.00033803773716917896, + "loss": 0.8677, + "step": 3790 + }, + { + "epoch": 0.3926431080801819, + "grad_norm": 1.414915680885315, + "learning_rate": 0.00033727003002382346, + "loss": 1.0965, + "step": 3800 + }, + { + "epoch": 0.393676379417235, + "grad_norm": 1.1397249698638916, + "learning_rate": 0.0003365013845761239, + "loss": 1.0057, + "step": 3810 + }, + { + "epoch": 0.3947096507542881, + "grad_norm": 1.470551609992981, + "learning_rate": 0.00033573180909033567, + "loss": 1.1104, + "step": 3820 + }, + { + "epoch": 0.3957429220913412, + "grad_norm": 0.9758129119873047, + "learning_rate": 0.0003349613118407138, + "loss": 1.0244, + "step": 3830 + }, + { + "epoch": 0.3967761934283943, + "grad_norm": 0.9483941793441772, + "learning_rate": 0.0003341899011114238, + "loss": 1.0993, + "step": 3840 + }, + { + "epoch": 0.39780946476544743, + "grad_norm": 1.0213124752044678, + "learning_rate": 0.00033341758519645273, + "loss": 0.9506, + "step": 3850 + }, + { + "epoch": 0.39884273610250054, + "grad_norm": 1.0135273933410645, + "learning_rate": 0.0003326443723995199, + "loss": 0.993, + "step": 3860 + }, + { + "epoch": 0.39987600743955365, + "grad_norm": 0.9621232748031616, + "learning_rate": 0.0003318702710339875, + "loss": 0.9969, + "step": 3870 + }, + { + "epoch": 0.40090927877660676, + "grad_norm": 1.1387019157409668, + "learning_rate": 0.0003310952894227718, + "loss": 1.0203, + "step": 3880 + }, + { + "epoch": 0.40194255011365987, + "grad_norm": 1.406870722770691, + "learning_rate": 0.0003303194358982526, + "loss": 1.0166, + "step": 3890 + }, + { + "epoch": 0.402975821450713, + "grad_norm": 1.339579701423645, + "learning_rate": 0.00032954271880218465, + "loss": 0.9749, + "step": 3900 + }, + { + "epoch": 0.4040090927877661, + "grad_norm": 1.2406803369522095, + "learning_rate": 0.00032876514648560757, + "loss": 0.943, + "step": 3910 + }, + { + "epoch": 0.4050423641248192, + "grad_norm": 1.172455072402954, + "learning_rate": 0.00032798672730875603, + "loss": 0.9874, + "step": 3920 + }, + { + "epoch": 0.4060756354618723, + "grad_norm": 1.2804698944091797, + "learning_rate": 0.0003272074696409697, + "loss": 0.8747, + "step": 3930 + }, + { + "epoch": 0.4071089067989254, + "grad_norm": 1.3023015260696411, + "learning_rate": 0.00032642738186060375, + "loss": 1.0686, + "step": 3940 + }, + { + "epoch": 0.40814217813597853, + "grad_norm": 1.0053542852401733, + "learning_rate": 0.00032564647235493833, + "loss": 1.074, + "step": 3950 + }, + { + "epoch": 0.40917544947303164, + "grad_norm": 1.0846545696258545, + "learning_rate": 0.0003248647495200886, + "loss": 1.0043, + "step": 3960 + }, + { + "epoch": 0.41020872081008475, + "grad_norm": 1.0012749433517456, + "learning_rate": 0.00032408222176091427, + "loss": 1.0302, + "step": 3970 + }, + { + "epoch": 0.41124199214713786, + "grad_norm": 1.074317455291748, + "learning_rate": 0.0003232988974909296, + "loss": 0.9389, + "step": 3980 + }, + { + "epoch": 0.41227526348419097, + "grad_norm": 1.5110340118408203, + "learning_rate": 0.00032251478513221254, + "loss": 0.9772, + "step": 3990 + }, + { + "epoch": 0.4133085348212441, + "grad_norm": 0.9564982652664185, + "learning_rate": 0.00032172989311531427, + "loss": 1.0754, + "step": 4000 + }, + { + "epoch": 0.4143418061582972, + "grad_norm": 1.091377854347229, + "learning_rate": 0.0003209442298791689, + "loss": 0.9343, + "step": 4010 + }, + { + "epoch": 0.4153750774953503, + "grad_norm": 1.3888299465179443, + "learning_rate": 0.00032015780387100227, + "loss": 0.9448, + "step": 4020 + }, + { + "epoch": 0.4164083488324034, + "grad_norm": 0.9059675335884094, + "learning_rate": 0.0003193706235462412, + "loss": 0.9877, + "step": 4030 + }, + { + "epoch": 0.4174416201694565, + "grad_norm": 1.5517827272415161, + "learning_rate": 0.0003185826973684231, + "loss": 1.0814, + "step": 4040 + }, + { + "epoch": 0.4184748915065096, + "grad_norm": 1.1768487691879272, + "learning_rate": 0.00031779403380910427, + "loss": 1.0654, + "step": 4050 + }, + { + "epoch": 0.41950816284356274, + "grad_norm": 1.0996865034103394, + "learning_rate": 0.0003170046413477692, + "loss": 1.0507, + "step": 4060 + }, + { + "epoch": 0.42054143418061585, + "grad_norm": 1.7401593923568726, + "learning_rate": 0.00031621452847173923, + "loss": 1.1611, + "step": 4070 + }, + { + "epoch": 0.42157470551766896, + "grad_norm": 1.4233949184417725, + "learning_rate": 0.00031542370367608176, + "loss": 1.0043, + "step": 4080 + }, + { + "epoch": 0.42260797685472207, + "grad_norm": 1.1471216678619385, + "learning_rate": 0.00031463217546351803, + "loss": 1.0389, + "step": 4090 + }, + { + "epoch": 0.4236412481917752, + "grad_norm": 0.9187898635864258, + "learning_rate": 0.0003138399523443326, + "loss": 0.9048, + "step": 4100 + }, + { + "epoch": 0.4246745195288283, + "grad_norm": 1.0712145566940308, + "learning_rate": 0.0003130470428362811, + "loss": 0.9538, + "step": 4110 + }, + { + "epoch": 0.4257077908658814, + "grad_norm": 1.3978677988052368, + "learning_rate": 0.0003122534554644995, + "loss": 0.9093, + "step": 4120 + }, + { + "epoch": 0.4267410622029345, + "grad_norm": 0.9700504541397095, + "learning_rate": 0.0003114591987614117, + "loss": 0.984, + "step": 4130 + }, + { + "epoch": 0.4277743335399876, + "grad_norm": 1.1521259546279907, + "learning_rate": 0.0003106642812666379, + "loss": 0.9965, + "step": 4140 + }, + { + "epoch": 0.4288076048770407, + "grad_norm": 1.2085431814193726, + "learning_rate": 0.0003098687115269034, + "loss": 0.9018, + "step": 4150 + }, + { + "epoch": 0.42984087621409384, + "grad_norm": 1.1607948541641235, + "learning_rate": 0.0003090724980959457, + "loss": 1.0776, + "step": 4160 + }, + { + "epoch": 0.43087414755114695, + "grad_norm": 1.4290366172790527, + "learning_rate": 0.0003082756495344234, + "loss": 0.9008, + "step": 4170 + }, + { + "epoch": 0.43190741888820006, + "grad_norm": 1.3393192291259766, + "learning_rate": 0.00030747817440982385, + "loss": 0.9175, + "step": 4180 + }, + { + "epoch": 0.43294069022525317, + "grad_norm": 1.4794620275497437, + "learning_rate": 0.00030668008129637104, + "loss": 1.0164, + "step": 4190 + }, + { + "epoch": 0.4339739615623063, + "grad_norm": 0.5630869269371033, + "learning_rate": 0.00030588137877493317, + "loss": 1.0511, + "step": 4200 + }, + { + "epoch": 0.4350072328993594, + "grad_norm": 1.0285252332687378, + "learning_rate": 0.0003050820754329309, + "loss": 0.8834, + "step": 4210 + }, + { + "epoch": 0.4360405042364125, + "grad_norm": 1.4534152746200562, + "learning_rate": 0.00030428217986424444, + "loss": 1.0248, + "step": 4220 + }, + { + "epoch": 0.4370737755734656, + "grad_norm": 1.7549463510513306, + "learning_rate": 0.0003034817006691219, + "loss": 0.9278, + "step": 4230 + }, + { + "epoch": 0.4381070469105187, + "grad_norm": 1.1008073091506958, + "learning_rate": 0.0003026806464540859, + "loss": 1.0172, + "step": 4240 + }, + { + "epoch": 0.4391403182475718, + "grad_norm": 1.042400598526001, + "learning_rate": 0.0003018790258318417, + "loss": 0.9703, + "step": 4250 + }, + { + "epoch": 0.44017358958462494, + "grad_norm": 1.428467035293579, + "learning_rate": 0.00030107684742118466, + "loss": 0.9187, + "step": 4260 + }, + { + "epoch": 0.44120686092167805, + "grad_norm": 0.8269082903862, + "learning_rate": 0.0003002741198469068, + "loss": 0.8501, + "step": 4270 + }, + { + "epoch": 0.44224013225873116, + "grad_norm": 1.0836302042007446, + "learning_rate": 0.00029947085173970506, + "loss": 0.9252, + "step": 4280 + }, + { + "epoch": 0.44327340359578427, + "grad_norm": 0.8747875690460205, + "learning_rate": 0.00029866705173608776, + "loss": 0.8576, + "step": 4290 + }, + { + "epoch": 0.4443066749328374, + "grad_norm": 1.261354923248291, + "learning_rate": 0.0002978627284782821, + "loss": 0.8515, + "step": 4300 + }, + { + "epoch": 0.4453399462698905, + "grad_norm": 1.2424492835998535, + "learning_rate": 0.0002970578906141411, + "loss": 0.9274, + "step": 4310 + }, + { + "epoch": 0.4463732176069436, + "grad_norm": 1.6803146600723267, + "learning_rate": 0.00029625254679705094, + "loss": 1.1236, + "step": 4320 + }, + { + "epoch": 0.4474064889439967, + "grad_norm": 1.284347653388977, + "learning_rate": 0.00029544670568583734, + "loss": 0.9698, + "step": 4330 + }, + { + "epoch": 0.4484397602810498, + "grad_norm": 0.7693723440170288, + "learning_rate": 0.00029464037594467284, + "loss": 0.9518, + "step": 4340 + }, + { + "epoch": 0.4494730316181029, + "grad_norm": 1.0315730571746826, + "learning_rate": 0.00029383356624298383, + "loss": 1.0873, + "step": 4350 + }, + { + "epoch": 0.45050630295515603, + "grad_norm": 0.8411365747451782, + "learning_rate": 0.0002930262852553568, + "loss": 1.0025, + "step": 4360 + }, + { + "epoch": 0.45153957429220914, + "grad_norm": 0.8453785181045532, + "learning_rate": 0.0002922185416614456, + "loss": 0.8811, + "step": 4370 + }, + { + "epoch": 0.45257284562926225, + "grad_norm": 1.2322771549224854, + "learning_rate": 0.0002914103441458775, + "loss": 1.0246, + "step": 4380 + }, + { + "epoch": 0.45360611696631536, + "grad_norm": 1.2167178392410278, + "learning_rate": 0.0002906017013981609, + "loss": 0.9645, + "step": 4390 + }, + { + "epoch": 0.4546393883033685, + "grad_norm": 1.0350911617279053, + "learning_rate": 0.0002897926221125906, + "loss": 0.9242, + "step": 4400 + }, + { + "epoch": 0.4556726596404216, + "grad_norm": 1.1375268697738647, + "learning_rate": 0.0002889831149881551, + "loss": 0.945, + "step": 4410 + }, + { + "epoch": 0.4567059309774747, + "grad_norm": 1.209620475769043, + "learning_rate": 0.0002881731887284429, + "loss": 1.0773, + "step": 4420 + }, + { + "epoch": 0.4577392023145278, + "grad_norm": 1.2932454347610474, + "learning_rate": 0.0002873628520415491, + "loss": 0.8517, + "step": 4430 + }, + { + "epoch": 0.4587724736515809, + "grad_norm": 1.0030730962753296, + "learning_rate": 0.0002865521136399814, + "loss": 0.8911, + "step": 4440 + }, + { + "epoch": 0.459805744988634, + "grad_norm": 0.9559937119483948, + "learning_rate": 0.0002857409822405666, + "loss": 0.9505, + "step": 4450 + }, + { + "epoch": 0.46083901632568713, + "grad_norm": 0.934845507144928, + "learning_rate": 0.00028492946656435704, + "loss": 1.0858, + "step": 4460 + }, + { + "epoch": 0.46187228766274024, + "grad_norm": 1.2203209400177002, + "learning_rate": 0.00028411757533653657, + "loss": 0.9919, + "step": 4470 + }, + { + "epoch": 0.46290555899979335, + "grad_norm": 1.3204402923583984, + "learning_rate": 0.000283305317286327, + "loss": 0.9198, + "step": 4480 + }, + { + "epoch": 0.46393883033684646, + "grad_norm": 0.7426913380622864, + "learning_rate": 0.0002824927011468939, + "loss": 1.0825, + "step": 4490 + }, + { + "epoch": 0.4649721016738996, + "grad_norm": 0.8991280198097229, + "learning_rate": 0.00028167973565525325, + "loss": 0.9528, + "step": 4500 + }, + { + "epoch": 0.4660053730109527, + "grad_norm": 0.697215735912323, + "learning_rate": 0.00028086642955217666, + "loss": 0.8118, + "step": 4510 + }, + { + "epoch": 0.4670386443480058, + "grad_norm": 1.4364171028137207, + "learning_rate": 0.0002800527915820984, + "loss": 0.8903, + "step": 4520 + }, + { + "epoch": 0.4680719156850589, + "grad_norm": 0.9677620530128479, + "learning_rate": 0.00027923883049302067, + "loss": 0.9083, + "step": 4530 + }, + { + "epoch": 0.469105187022112, + "grad_norm": 1.240580439567566, + "learning_rate": 0.00027842455503641965, + "loss": 0.8643, + "step": 4540 + }, + { + "epoch": 0.4701384583591651, + "grad_norm": 1.102919101715088, + "learning_rate": 0.00027760997396715167, + "loss": 1.0802, + "step": 4550 + }, + { + "epoch": 0.47117172969621823, + "grad_norm": 0.921911895275116, + "learning_rate": 0.0002767950960433588, + "loss": 0.8717, + "step": 4560 + }, + { + "epoch": 0.47220500103327134, + "grad_norm": 1.3429372310638428, + "learning_rate": 0.000275979930026375, + "loss": 0.9584, + "step": 4570 + }, + { + "epoch": 0.47323827237032445, + "grad_norm": 1.0819973945617676, + "learning_rate": 0.0002751644846806315, + "loss": 0.7652, + "step": 4580 + }, + { + "epoch": 0.47427154370737756, + "grad_norm": 0.9730139970779419, + "learning_rate": 0.00027434876877356287, + "loss": 0.8873, + "step": 4590 + }, + { + "epoch": 0.4753048150444307, + "grad_norm": 0.9837844371795654, + "learning_rate": 0.00027353279107551276, + "loss": 0.9581, + "step": 4600 + }, + { + "epoch": 0.4763380863814838, + "grad_norm": 1.1134988069534302, + "learning_rate": 0.0002727165603596394, + "loss": 0.9409, + "step": 4610 + }, + { + "epoch": 0.4773713577185369, + "grad_norm": 0.7778913378715515, + "learning_rate": 0.00027190008540182135, + "loss": 0.9099, + "step": 4620 + }, + { + "epoch": 0.47840462905559, + "grad_norm": 1.166568636894226, + "learning_rate": 0.00027108337498056356, + "loss": 0.8565, + "step": 4630 + }, + { + "epoch": 0.4794379003926431, + "grad_norm": 0.7446126341819763, + "learning_rate": 0.0002702664378769021, + "loss": 0.8251, + "step": 4640 + }, + { + "epoch": 0.4804711717296962, + "grad_norm": 1.5371274948120117, + "learning_rate": 0.0002694492828743106, + "loss": 0.9475, + "step": 4650 + }, + { + "epoch": 0.48150444306674933, + "grad_norm": 0.7517269253730774, + "learning_rate": 0.0002686319187586054, + "loss": 0.8798, + "step": 4660 + }, + { + "epoch": 0.48253771440380244, + "grad_norm": 1.4352338314056396, + "learning_rate": 0.00026781435431785116, + "loss": 1.0462, + "step": 4670 + }, + { + "epoch": 0.48357098574085555, + "grad_norm": 1.1265496015548706, + "learning_rate": 0.0002669965983422663, + "loss": 0.9107, + "step": 4680 + }, + { + "epoch": 0.48460425707790866, + "grad_norm": 0.8634279370307922, + "learning_rate": 0.00026617865962412865, + "loss": 0.8822, + "step": 4690 + }, + { + "epoch": 0.48563752841496177, + "grad_norm": 1.0112096071243286, + "learning_rate": 0.0002653605469576809, + "loss": 0.8842, + "step": 4700 + }, + { + "epoch": 0.4866707997520149, + "grad_norm": 0.8251860737800598, + "learning_rate": 0.0002645422691390358, + "loss": 0.908, + "step": 4710 + }, + { + "epoch": 0.487704071089068, + "grad_norm": 0.7483717203140259, + "learning_rate": 0.0002637238349660819, + "loss": 0.8912, + "step": 4720 + }, + { + "epoch": 0.4887373424261211, + "grad_norm": 0.8353511691093445, + "learning_rate": 0.0002629052532383888, + "loss": 0.7702, + "step": 4730 + }, + { + "epoch": 0.4897706137631742, + "grad_norm": 0.9910038113594055, + "learning_rate": 0.00026208653275711265, + "loss": 0.8797, + "step": 4740 + }, + { + "epoch": 0.4908038851002273, + "grad_norm": 1.1162399053573608, + "learning_rate": 0.00026126768232490115, + "loss": 0.9161, + "step": 4750 + }, + { + "epoch": 0.49183715643728043, + "grad_norm": 0.8416900038719177, + "learning_rate": 0.00026044871074579955, + "loss": 1.0406, + "step": 4760 + }, + { + "epoch": 0.49287042777433354, + "grad_norm": 1.2481132745742798, + "learning_rate": 0.0002596296268251556, + "loss": 1.0545, + "step": 4770 + }, + { + "epoch": 0.49390369911138665, + "grad_norm": 1.115768551826477, + "learning_rate": 0.0002588104393695245, + "loss": 0.9128, + "step": 4780 + }, + { + "epoch": 0.49493697044843976, + "grad_norm": 0.7396090030670166, + "learning_rate": 0.0002579911571865752, + "loss": 1.0822, + "step": 4790 + }, + { + "epoch": 0.49597024178549287, + "grad_norm": 0.9519025087356567, + "learning_rate": 0.0002571717890849946, + "loss": 0.8659, + "step": 4800 + }, + { + "epoch": 0.497003513122546, + "grad_norm": 0.7343184947967529, + "learning_rate": 0.0002563523438743939, + "loss": 0.8773, + "step": 4810 + }, + { + "epoch": 0.4980367844595991, + "grad_norm": 1.4175959825515747, + "learning_rate": 0.0002555328303652129, + "loss": 0.7956, + "step": 4820 + }, + { + "epoch": 0.4990700557966522, + "grad_norm": 1.0520236492156982, + "learning_rate": 0.000254713257368626, + "loss": 0.9604, + "step": 4830 + }, + { + "epoch": 0.5001033271337053, + "grad_norm": 0.7733720541000366, + "learning_rate": 0.0002538936336964471, + "loss": 0.7715, + "step": 4840 + }, + { + "epoch": 0.5011365984707584, + "grad_norm": 1.4437015056610107, + "learning_rate": 0.0002530739681610349, + "loss": 0.9329, + "step": 4850 + }, + { + "epoch": 0.5021698698078115, + "grad_norm": 0.892091691493988, + "learning_rate": 0.00025225426957519825, + "loss": 0.8434, + "step": 4860 + }, + { + "epoch": 0.5032031411448646, + "grad_norm": 0.9304814338684082, + "learning_rate": 0.0002514345467521014, + "loss": 0.8912, + "step": 4870 + }, + { + "epoch": 0.5042364124819178, + "grad_norm": 1.063408374786377, + "learning_rate": 0.00025061480850516914, + "loss": 0.9901, + "step": 4880 + }, + { + "epoch": 0.5052696838189709, + "grad_norm": 1.0481808185577393, + "learning_rate": 0.00024979506364799207, + "loss": 0.89, + "step": 4890 + }, + { + "epoch": 0.506302955156024, + "grad_norm": 1.2767094373703003, + "learning_rate": 0.000248975320994232, + "loss": 1.1239, + "step": 4900 + }, + { + "epoch": 0.5073362264930771, + "grad_norm": 1.0354382991790771, + "learning_rate": 0.00024815558935752677, + "loss": 0.8866, + "step": 4910 + }, + { + "epoch": 0.5083694978301302, + "grad_norm": 0.7483660578727722, + "learning_rate": 0.0002473358775513959, + "loss": 0.8817, + "step": 4920 + }, + { + "epoch": 0.5094027691671833, + "grad_norm": 0.6301681399345398, + "learning_rate": 0.000246516194389146, + "loss": 0.8991, + "step": 4930 + }, + { + "epoch": 0.5104360405042364, + "grad_norm": 1.0671608448028564, + "learning_rate": 0.0002456965486837752, + "loss": 0.7891, + "step": 4940 + }, + { + "epoch": 0.5114693118412895, + "grad_norm": 1.3131459951400757, + "learning_rate": 0.00024487694924787935, + "loss": 0.9388, + "step": 4950 + }, + { + "epoch": 0.5125025831783426, + "grad_norm": 1.2082393169403076, + "learning_rate": 0.00024405740489355634, + "loss": 0.8558, + "step": 4960 + }, + { + "epoch": 0.5135358545153957, + "grad_norm": 0.915107786655426, + "learning_rate": 0.0002432379244323124, + "loss": 0.8703, + "step": 4970 + }, + { + "epoch": 0.5145691258524488, + "grad_norm": 1.2518354654312134, + "learning_rate": 0.00024241851667496645, + "loss": 0.9959, + "step": 4980 + }, + { + "epoch": 0.515602397189502, + "grad_norm": 1.0495281219482422, + "learning_rate": 0.00024159919043155556, + "loss": 1.0497, + "step": 4990 + }, + { + "epoch": 0.5166356685265551, + "grad_norm": 1.469854712486267, + "learning_rate": 0.0002407799545112409, + "loss": 0.7946, + "step": 5000 + }, + { + "epoch": 0.5176689398636082, + "grad_norm": 0.7165640592575073, + "learning_rate": 0.0002399608177222122, + "loss": 0.7768, + "step": 5010 + }, + { + "epoch": 0.5187022112006613, + "grad_norm": 0.7148188352584839, + "learning_rate": 0.00023914178887159327, + "loss": 0.8901, + "step": 5020 + }, + { + "epoch": 0.5197354825377144, + "grad_norm": 1.2880741357803345, + "learning_rate": 0.0002383228767653474, + "loss": 1.0241, + "step": 5030 + }, + { + "epoch": 0.5207687538747675, + "grad_norm": 1.3181527853012085, + "learning_rate": 0.0002375040902081831, + "loss": 0.9248, + "step": 5040 + }, + { + "epoch": 0.5218020252118206, + "grad_norm": 1.5515258312225342, + "learning_rate": 0.00023668543800345852, + "loss": 0.8286, + "step": 5050 + }, + { + "epoch": 0.5228352965488737, + "grad_norm": 1.0438510179519653, + "learning_rate": 0.0002358669289530875, + "loss": 0.7546, + "step": 5060 + }, + { + "epoch": 0.5238685678859268, + "grad_norm": 1.179828405380249, + "learning_rate": 0.00023504857185744453, + "loss": 0.8099, + "step": 5070 + }, + { + "epoch": 0.52490183922298, + "grad_norm": 1.0748212337493896, + "learning_rate": 0.00023423037551527085, + "loss": 0.8692, + "step": 5080 + }, + { + "epoch": 0.5259351105600331, + "grad_norm": 0.8671577572822571, + "learning_rate": 0.00023341234872357872, + "loss": 0.8423, + "step": 5090 + }, + { + "epoch": 0.5269683818970862, + "grad_norm": 1.147182583808899, + "learning_rate": 0.00023259450027755764, + "loss": 0.8823, + "step": 5100 + }, + { + "epoch": 0.5280016532341393, + "grad_norm": 1.0983150005340576, + "learning_rate": 0.0002317768389704799, + "loss": 0.879, + "step": 5110 + }, + { + "epoch": 0.5290349245711924, + "grad_norm": 1.5440839529037476, + "learning_rate": 0.00023095937359360525, + "loss": 0.9303, + "step": 5120 + }, + { + "epoch": 0.5300681959082455, + "grad_norm": 1.0175645351409912, + "learning_rate": 0.00023014211293608718, + "loss": 0.8823, + "step": 5130 + }, + { + "epoch": 0.5311014672452986, + "grad_norm": 1.1853641271591187, + "learning_rate": 0.00022932506578487782, + "loss": 0.9467, + "step": 5140 + }, + { + "epoch": 0.5321347385823517, + "grad_norm": 0.9132509827613831, + "learning_rate": 0.00022850824092463413, + "loss": 0.7992, + "step": 5150 + }, + { + "epoch": 0.5331680099194048, + "grad_norm": 0.8958359360694885, + "learning_rate": 0.0002276916471376227, + "loss": 0.9003, + "step": 5160 + }, + { + "epoch": 0.5342012812564579, + "grad_norm": 0.9055427312850952, + "learning_rate": 0.00022687529320362587, + "loss": 0.8922, + "step": 5170 + }, + { + "epoch": 0.535234552593511, + "grad_norm": 1.2413357496261597, + "learning_rate": 0.00022605918789984707, + "loss": 0.8021, + "step": 5180 + }, + { + "epoch": 0.5362678239305642, + "grad_norm": 1.096956729888916, + "learning_rate": 0.00022524334000081664, + "loss": 0.7335, + "step": 5190 + }, + { + "epoch": 0.5373010952676173, + "grad_norm": 1.0181100368499756, + "learning_rate": 0.00022442775827829722, + "loss": 0.9217, + "step": 5200 + }, + { + "epoch": 0.5383343666046704, + "grad_norm": 0.9444778561592102, + "learning_rate": 0.0002236124515011897, + "loss": 0.8444, + "step": 5210 + }, + { + "epoch": 0.5393676379417235, + "grad_norm": 0.9442963004112244, + "learning_rate": 0.00022279742843543887, + "loss": 0.7901, + "step": 5220 + }, + { + "epoch": 0.5404009092787766, + "grad_norm": 1.3280583620071411, + "learning_rate": 0.00022198269784393904, + "loss": 0.9, + "step": 5230 + }, + { + "epoch": 0.5414341806158297, + "grad_norm": 0.7821478843688965, + "learning_rate": 0.00022116826848643994, + "loss": 0.8782, + "step": 5240 + }, + { + "epoch": 0.5424674519528828, + "grad_norm": 0.942866861820221, + "learning_rate": 0.00022035414911945252, + "loss": 0.8268, + "step": 5250 + }, + { + "epoch": 0.5435007232899359, + "grad_norm": 1.0075554847717285, + "learning_rate": 0.0002195403484961549, + "loss": 0.7923, + "step": 5260 + }, + { + "epoch": 0.544533994626989, + "grad_norm": 1.0974137783050537, + "learning_rate": 0.00021872687536629806, + "loss": 0.8108, + "step": 5270 + }, + { + "epoch": 0.5455672659640421, + "grad_norm": 0.8785361647605896, + "learning_rate": 0.00021791373847611186, + "loss": 0.8236, + "step": 5280 + }, + { + "epoch": 0.5466005373010953, + "grad_norm": 1.300614356994629, + "learning_rate": 0.00021710094656821098, + "loss": 0.9875, + "step": 5290 + }, + { + "epoch": 0.5476338086381484, + "grad_norm": 0.9605757594108582, + "learning_rate": 0.0002162885083815011, + "loss": 0.8824, + "step": 5300 + }, + { + "epoch": 0.5486670799752015, + "grad_norm": 0.8986537456512451, + "learning_rate": 0.0002154764326510847, + "loss": 0.9041, + "step": 5310 + }, + { + "epoch": 0.5497003513122546, + "grad_norm": 0.8150350451469421, + "learning_rate": 0.0002146647281081671, + "loss": 0.8553, + "step": 5320 + }, + { + "epoch": 0.5507336226493077, + "grad_norm": 1.396665334701538, + "learning_rate": 0.00021385340347996297, + "loss": 0.9266, + "step": 5330 + }, + { + "epoch": 0.5517668939863608, + "grad_norm": 1.1270840167999268, + "learning_rate": 0.00021304246748960208, + "loss": 0.8426, + "step": 5340 + }, + { + "epoch": 0.5528001653234139, + "grad_norm": 1.4210362434387207, + "learning_rate": 0.00021223192885603571, + "loss": 0.9085, + "step": 5350 + }, + { + "epoch": 0.553833436660467, + "grad_norm": 0.9129871726036072, + "learning_rate": 0.00021142179629394283, + "loss": 0.8978, + "step": 5360 + }, + { + "epoch": 0.5548667079975201, + "grad_norm": 1.0395097732543945, + "learning_rate": 0.00021061207851363661, + "loss": 0.9224, + "step": 5370 + }, + { + "epoch": 0.5558999793345732, + "grad_norm": 0.5978155136108398, + "learning_rate": 0.00020980278422097045, + "loss": 0.789, + "step": 5380 + }, + { + "epoch": 0.5569332506716264, + "grad_norm": 0.8347190022468567, + "learning_rate": 0.0002089939221172446, + "loss": 0.8651, + "step": 5390 + }, + { + "epoch": 0.5579665220086795, + "grad_norm": 1.0471965074539185, + "learning_rate": 0.00020818550089911234, + "loss": 0.9259, + "step": 5400 + }, + { + "epoch": 0.5589997933457326, + "grad_norm": 0.8594735264778137, + "learning_rate": 0.00020737752925848707, + "loss": 0.7926, + "step": 5410 + }, + { + "epoch": 0.5600330646827857, + "grad_norm": 0.9445158243179321, + "learning_rate": 0.00020657001588244816, + "loss": 0.9561, + "step": 5420 + }, + { + "epoch": 0.5610663360198388, + "grad_norm": 0.7682790160179138, + "learning_rate": 0.00020576296945314787, + "loss": 0.8636, + "step": 5430 + }, + { + "epoch": 0.5620996073568919, + "grad_norm": 0.8511012196540833, + "learning_rate": 0.00020495639864771814, + "loss": 0.877, + "step": 5440 + }, + { + "epoch": 0.563132878693945, + "grad_norm": 1.3375914096832275, + "learning_rate": 0.00020415031213817704, + "loss": 0.8865, + "step": 5450 + }, + { + "epoch": 0.5641661500309981, + "grad_norm": 1.4135067462921143, + "learning_rate": 0.0002033447185913356, + "loss": 0.8816, + "step": 5460 + }, + { + "epoch": 0.5651994213680512, + "grad_norm": 1.0292867422103882, + "learning_rate": 0.0002025396266687045, + "loss": 0.7509, + "step": 5470 + }, + { + "epoch": 0.5662326927051043, + "grad_norm": 0.8321288228034973, + "learning_rate": 0.00020173504502640164, + "loss": 0.8376, + "step": 5480 + }, + { + "epoch": 0.5672659640421575, + "grad_norm": 0.7995291948318481, + "learning_rate": 0.000200930982315058, + "loss": 0.822, + "step": 5490 + }, + { + "epoch": 0.5682992353792106, + "grad_norm": 0.7237703800201416, + "learning_rate": 0.0002001274471797252, + "loss": 0.9819, + "step": 5500 + }, + { + "epoch": 0.5693325067162637, + "grad_norm": 1.2666767835617065, + "learning_rate": 0.0001993244482597829, + "loss": 0.7477, + "step": 5510 + }, + { + "epoch": 0.5703657780533168, + "grad_norm": 1.1628350019454956, + "learning_rate": 0.00019852199418884527, + "loss": 0.7239, + "step": 5520 + }, + { + "epoch": 0.5713990493903699, + "grad_norm": 1.1754180192947388, + "learning_rate": 0.00019772009359466828, + "loss": 0.8326, + "step": 5530 + }, + { + "epoch": 0.572432320727423, + "grad_norm": 0.9641863703727722, + "learning_rate": 0.00019691875509905718, + "loss": 0.779, + "step": 5540 + }, + { + "epoch": 0.5734655920644761, + "grad_norm": 1.4042216539382935, + "learning_rate": 0.00019611798731777397, + "loss": 0.8895, + "step": 5550 + }, + { + "epoch": 0.5744988634015292, + "grad_norm": 0.8444200158119202, + "learning_rate": 0.00019531779886044424, + "loss": 0.8636, + "step": 5560 + }, + { + "epoch": 0.5755321347385823, + "grad_norm": 0.8476575016975403, + "learning_rate": 0.0001945181983304647, + "loss": 0.802, + "step": 5570 + }, + { + "epoch": 0.5765654060756354, + "grad_norm": 1.086524486541748, + "learning_rate": 0.00019371919432491108, + "loss": 0.8356, + "step": 5580 + }, + { + "epoch": 0.5775986774126886, + "grad_norm": 0.8224913477897644, + "learning_rate": 0.00019292079543444557, + "loss": 0.8772, + "step": 5590 + }, + { + "epoch": 0.5786319487497417, + "grad_norm": 0.8469100594520569, + "learning_rate": 0.00019212301024322396, + "loss": 0.883, + "step": 5600 + }, + { + "epoch": 0.5796652200867948, + "grad_norm": 0.6162629127502441, + "learning_rate": 0.00019132584732880383, + "loss": 0.7449, + "step": 5610 + }, + { + "epoch": 0.5806984914238479, + "grad_norm": 1.1471900939941406, + "learning_rate": 0.00019052931526205267, + "loss": 0.8848, + "step": 5620 + }, + { + "epoch": 0.581731762760901, + "grad_norm": 1.4094953536987305, + "learning_rate": 0.00018973342260705458, + "loss": 0.7942, + "step": 5630 + }, + { + "epoch": 0.5827650340979541, + "grad_norm": 0.8406280279159546, + "learning_rate": 0.00018893817792101935, + "loss": 0.8205, + "step": 5640 + }, + { + "epoch": 0.5837983054350072, + "grad_norm": 1.5820422172546387, + "learning_rate": 0.00018814358975418984, + "loss": 0.733, + "step": 5650 + }, + { + "epoch": 0.5848315767720603, + "grad_norm": 1.0317678451538086, + "learning_rate": 0.00018734966664975023, + "loss": 0.8889, + "step": 5660 + }, + { + "epoch": 0.5858648481091134, + "grad_norm": 1.1359983682632446, + "learning_rate": 0.00018655641714373413, + "loss": 0.897, + "step": 5670 + }, + { + "epoch": 0.5868981194461665, + "grad_norm": 1.0122281312942505, + "learning_rate": 0.00018576384976493282, + "loss": 0.7574, + "step": 5680 + }, + { + "epoch": 0.5879313907832197, + "grad_norm": 0.9288797378540039, + "learning_rate": 0.00018497197303480348, + "loss": 0.7553, + "step": 5690 + }, + { + "epoch": 0.5889646621202728, + "grad_norm": 0.8775319457054138, + "learning_rate": 0.00018418079546737783, + "loss": 0.74, + "step": 5700 + }, + { + "epoch": 0.5899979334573259, + "grad_norm": 1.075302243232727, + "learning_rate": 0.0001833903255691702, + "loss": 0.8761, + "step": 5710 + }, + { + "epoch": 0.591031204794379, + "grad_norm": 1.0737923383712769, + "learning_rate": 0.00018260057183908625, + "loss": 0.8817, + "step": 5720 + }, + { + "epoch": 0.5920644761314321, + "grad_norm": 1.1469508409500122, + "learning_rate": 0.00018181154276833178, + "loss": 0.7806, + "step": 5730 + }, + { + "epoch": 0.5930977474684852, + "grad_norm": 0.8804445266723633, + "learning_rate": 0.00018102324684032115, + "loss": 0.7683, + "step": 5740 + }, + { + "epoch": 0.5941310188055383, + "grad_norm": 0.9593983888626099, + "learning_rate": 0.00018023569253058613, + "loss": 0.8427, + "step": 5750 + }, + { + "epoch": 0.5951642901425914, + "grad_norm": 0.7528694272041321, + "learning_rate": 0.00017944888830668488, + "loss": 0.8697, + "step": 5760 + }, + { + "epoch": 0.5961975614796445, + "grad_norm": 1.0688844919204712, + "learning_rate": 0.00017866284262811093, + "loss": 0.779, + "step": 5770 + }, + { + "epoch": 0.5972308328166976, + "grad_norm": 1.181410312652588, + "learning_rate": 0.00017787756394620202, + "loss": 0.8387, + "step": 5780 + }, + { + "epoch": 0.5982641041537508, + "grad_norm": 0.7557913064956665, + "learning_rate": 0.0001770930607040494, + "loss": 0.7897, + "step": 5790 + }, + { + "epoch": 0.5992973754908039, + "grad_norm": 0.9230046272277832, + "learning_rate": 0.00017630934133640704, + "loss": 0.7056, + "step": 5800 + }, + { + "epoch": 0.600330646827857, + "grad_norm": 0.839525043964386, + "learning_rate": 0.00017552641426960092, + "loss": 0.8133, + "step": 5810 + }, + { + "epoch": 0.6013639181649101, + "grad_norm": 1.102609395980835, + "learning_rate": 0.0001747442879214384, + "loss": 0.884, + "step": 5820 + }, + { + "epoch": 0.6023971895019632, + "grad_norm": 0.8469570875167847, + "learning_rate": 0.0001739629707011177, + "loss": 0.7398, + "step": 5830 + }, + { + "epoch": 0.6034304608390163, + "grad_norm": 1.2042063474655151, + "learning_rate": 0.0001731824710091377, + "loss": 0.8074, + "step": 5840 + }, + { + "epoch": 0.6044637321760694, + "grad_norm": 2.6056320667266846, + "learning_rate": 0.00017240279723720732, + "loss": 0.8092, + "step": 5850 + }, + { + "epoch": 0.6054970035131225, + "grad_norm": 0.788226306438446, + "learning_rate": 0.00017162395776815532, + "loss": 0.8346, + "step": 5860 + }, + { + "epoch": 0.6065302748501756, + "grad_norm": 0.8313378095626831, + "learning_rate": 0.00017084596097584044, + "loss": 0.7632, + "step": 5870 + }, + { + "epoch": 0.6075635461872287, + "grad_norm": 0.9668551087379456, + "learning_rate": 0.00017006881522506123, + "loss": 0.7813, + "step": 5880 + }, + { + "epoch": 0.6085968175242819, + "grad_norm": 1.2980097532272339, + "learning_rate": 0.00016929252887146597, + "loss": 0.633, + "step": 5890 + }, + { + "epoch": 0.609630088861335, + "grad_norm": 1.296339511871338, + "learning_rate": 0.00016851711026146294, + "loss": 0.8067, + "step": 5900 + }, + { + "epoch": 0.6106633601983881, + "grad_norm": 0.8966313600540161, + "learning_rate": 0.00016774256773213075, + "loss": 0.8509, + "step": 5910 + }, + { + "epoch": 0.6116966315354412, + "grad_norm": 1.408347249031067, + "learning_rate": 0.00016696890961112867, + "loss": 0.6981, + "step": 5920 + }, + { + "epoch": 0.6127299028724943, + "grad_norm": 1.421396017074585, + "learning_rate": 0.00016619614421660702, + "loss": 0.7626, + "step": 5930 + }, + { + "epoch": 0.6137631742095474, + "grad_norm": 1.4539517164230347, + "learning_rate": 0.0001654242798571177, + "loss": 0.7558, + "step": 5940 + }, + { + "epoch": 0.6147964455466005, + "grad_norm": 1.0731654167175293, + "learning_rate": 0.0001646533248315252, + "loss": 0.7626, + "step": 5950 + }, + { + "epoch": 0.6158297168836536, + "grad_norm": 1.100475788116455, + "learning_rate": 0.00016388328742891677, + "loss": 0.7243, + "step": 5960 + }, + { + "epoch": 0.6168629882207067, + "grad_norm": 1.0175180435180664, + "learning_rate": 0.00016311417592851396, + "loss": 0.8071, + "step": 5970 + }, + { + "epoch": 0.6178962595577598, + "grad_norm": 0.9055405259132385, + "learning_rate": 0.00016234599859958287, + "loss": 0.7203, + "step": 5980 + }, + { + "epoch": 0.618929530894813, + "grad_norm": 1.0334464311599731, + "learning_rate": 0.00016157876370134617, + "loss": 0.706, + "step": 5990 + }, + { + "epoch": 0.6199628022318661, + "grad_norm": 1.1770230531692505, + "learning_rate": 0.00016081247948289353, + "loss": 0.7802, + "step": 6000 + }, + { + "epoch": 0.6209960735689192, + "grad_norm": 0.8190944790840149, + "learning_rate": 0.00016004715418309304, + "loss": 0.7762, + "step": 6010 + }, + { + "epoch": 0.6220293449059723, + "grad_norm": 1.0050560235977173, + "learning_rate": 0.0001592827960305029, + "loss": 0.7886, + "step": 6020 + }, + { + "epoch": 0.6230626162430254, + "grad_norm": 1.1464440822601318, + "learning_rate": 0.000158519413243283, + "loss": 0.8016, + "step": 6030 + }, + { + "epoch": 0.6240958875800785, + "grad_norm": 0.8896029591560364, + "learning_rate": 0.00015775701402910626, + "loss": 0.802, + "step": 6040 + }, + { + "epoch": 0.6251291589171316, + "grad_norm": 0.7307144403457642, + "learning_rate": 0.00015699560658507012, + "loss": 0.7755, + "step": 6050 + }, + { + "epoch": 0.6261624302541847, + "grad_norm": 0.767782986164093, + "learning_rate": 0.00015623519909760953, + "loss": 0.697, + "step": 6060 + }, + { + "epoch": 0.6271957015912378, + "grad_norm": 0.8861032724380493, + "learning_rate": 0.00015547579974240766, + "loss": 0.8005, + "step": 6070 + }, + { + "epoch": 0.6282289729282909, + "grad_norm": 0.9863024950027466, + "learning_rate": 0.00015471741668430863, + "loss": 0.8324, + "step": 6080 + }, + { + "epoch": 0.629262244265344, + "grad_norm": 0.8046024441719055, + "learning_rate": 0.0001539600580772296, + "loss": 0.8615, + "step": 6090 + }, + { + "epoch": 0.6302955156023972, + "grad_norm": 1.283008098602295, + "learning_rate": 0.0001532037320640734, + "loss": 0.8317, + "step": 6100 + }, + { + "epoch": 0.6313287869394503, + "grad_norm": 0.9851896166801453, + "learning_rate": 0.00015244844677664036, + "loss": 0.7516, + "step": 6110 + }, + { + "epoch": 0.6323620582765034, + "grad_norm": 1.4205191135406494, + "learning_rate": 0.00015169421033554137, + "loss": 0.7692, + "step": 6120 + }, + { + "epoch": 0.6333953296135565, + "grad_norm": 0.9991669058799744, + "learning_rate": 0.0001509410308501104, + "loss": 0.7453, + "step": 6130 + }, + { + "epoch": 0.6344286009506096, + "grad_norm": 0.723731279373169, + "learning_rate": 0.00015018891641831744, + "loss": 0.8026, + "step": 6140 + }, + { + "epoch": 0.6354618722876627, + "grad_norm": 0.5217877626419067, + "learning_rate": 0.00014943787512668118, + "loss": 0.7759, + "step": 6150 + }, + { + "epoch": 0.6364951436247158, + "grad_norm": 0.6816266179084778, + "learning_rate": 0.0001486879150501822, + "loss": 0.7659, + "step": 6160 + }, + { + "epoch": 0.6375284149617689, + "grad_norm": 0.5098093152046204, + "learning_rate": 0.00014793904425217648, + "loss": 0.8352, + "step": 6170 + }, + { + "epoch": 0.638561686298822, + "grad_norm": 1.0227922201156616, + "learning_rate": 0.00014719127078430795, + "loss": 0.8162, + "step": 6180 + }, + { + "epoch": 0.6395949576358751, + "grad_norm": 1.5966726541519165, + "learning_rate": 0.00014644460268642263, + "loss": 0.7707, + "step": 6190 + }, + { + "epoch": 0.6406282289729283, + "grad_norm": 0.6314913630485535, + "learning_rate": 0.00014569904798648176, + "loss": 0.7286, + "step": 6200 + }, + { + "epoch": 0.6416615003099814, + "grad_norm": 1.072977900505066, + "learning_rate": 0.00014495461470047594, + "loss": 0.7554, + "step": 6210 + }, + { + "epoch": 0.6426947716470345, + "grad_norm": 1.458894968032837, + "learning_rate": 0.0001442113108323383, + "loss": 0.8975, + "step": 6220 + }, + { + "epoch": 0.6437280429840876, + "grad_norm": 0.7281268239021301, + "learning_rate": 0.00014346914437385905, + "loss": 0.6558, + "step": 6230 + }, + { + "epoch": 0.6447613143211407, + "grad_norm": 0.9581575393676758, + "learning_rate": 0.00014272812330459893, + "loss": 0.7082, + "step": 6240 + }, + { + "epoch": 0.6457945856581938, + "grad_norm": 1.271925687789917, + "learning_rate": 0.00014198825559180432, + "loss": 0.8919, + "step": 6250 + }, + { + "epoch": 0.6468278569952469, + "grad_norm": 0.9335398077964783, + "learning_rate": 0.00014124954919032064, + "loss": 0.7847, + "step": 6260 + }, + { + "epoch": 0.6478611283323, + "grad_norm": 0.7705923318862915, + "learning_rate": 0.00014051201204250726, + "loss": 0.8197, + "step": 6270 + }, + { + "epoch": 0.6488943996693531, + "grad_norm": 1.433953881263733, + "learning_rate": 0.00013977565207815238, + "loss": 0.8028, + "step": 6280 + }, + { + "epoch": 0.6499276710064062, + "grad_norm": 1.4676005840301514, + "learning_rate": 0.0001390404772143872, + "loss": 0.6803, + "step": 6290 + }, + { + "epoch": 0.6509609423434594, + "grad_norm": 0.8155161738395691, + "learning_rate": 0.00013830649535560093, + "loss": 0.8868, + "step": 6300 + }, + { + "epoch": 0.6519942136805125, + "grad_norm": 0.799152672290802, + "learning_rate": 0.00013757371439335604, + "loss": 0.716, + "step": 6310 + }, + { + "epoch": 0.6530274850175656, + "grad_norm": 1.120835304260254, + "learning_rate": 0.0001368421422063036, + "loss": 0.7953, + "step": 6320 + }, + { + "epoch": 0.6540607563546187, + "grad_norm": 0.7950730919837952, + "learning_rate": 0.00013611178666009794, + "loss": 0.7389, + "step": 6330 + }, + { + "epoch": 0.6550940276916718, + "grad_norm": 0.9817484617233276, + "learning_rate": 0.0001353826556073125, + "loss": 0.7451, + "step": 6340 + }, + { + "epoch": 0.6561272990287249, + "grad_norm": 0.7330102324485779, + "learning_rate": 0.00013465475688735526, + "loss": 0.7386, + "step": 6350 + }, + { + "epoch": 0.657160570365778, + "grad_norm": 0.7877906560897827, + "learning_rate": 0.00013392809832638486, + "loss": 0.7596, + "step": 6360 + }, + { + "epoch": 0.6581938417028311, + "grad_norm": 0.6060128211975098, + "learning_rate": 0.00013320268773722553, + "loss": 0.6482, + "step": 6370 + }, + { + "epoch": 0.6592271130398842, + "grad_norm": 1.8573722839355469, + "learning_rate": 0.00013247853291928395, + "loss": 0.7416, + "step": 6380 + }, + { + "epoch": 0.6602603843769373, + "grad_norm": 0.8654054403305054, + "learning_rate": 0.00013175564165846532, + "loss": 0.7027, + "step": 6390 + }, + { + "epoch": 0.6612936557139905, + "grad_norm": 0.7953044772148132, + "learning_rate": 0.00013103402172708918, + "loss": 0.7331, + "step": 6400 + }, + { + "epoch": 0.6623269270510436, + "grad_norm": 0.8240858316421509, + "learning_rate": 0.0001303136808838061, + "loss": 0.7007, + "step": 6410 + }, + { + "epoch": 0.6633601983880967, + "grad_norm": 0.8028026819229126, + "learning_rate": 0.00012959462687351435, + "loss": 0.7626, + "step": 6420 + }, + { + "epoch": 0.6643934697251498, + "grad_norm": 1.4915801286697388, + "learning_rate": 0.0001288768674272765, + "loss": 0.8101, + "step": 6430 + }, + { + "epoch": 0.6654267410622029, + "grad_norm": 0.8312230110168457, + "learning_rate": 0.0001281604102622364, + "loss": 0.7186, + "step": 6440 + }, + { + "epoch": 0.666460012399256, + "grad_norm": 0.8355895280838013, + "learning_rate": 0.000127445263081536, + "loss": 0.756, + "step": 6450 + }, + { + "epoch": 0.6674932837363091, + "grad_norm": 0.7141729593276978, + "learning_rate": 0.0001267314335742327, + "loss": 0.707, + "step": 6460 + }, + { + "epoch": 0.6685265550733622, + "grad_norm": 1.2061463594436646, + "learning_rate": 0.000126018929415217, + "loss": 0.6433, + "step": 6470 + }, + { + "epoch": 0.6695598264104153, + "grad_norm": 0.9426934123039246, + "learning_rate": 0.00012530775826512916, + "loss": 0.7131, + "step": 6480 + }, + { + "epoch": 0.6705930977474684, + "grad_norm": 0.5171589255332947, + "learning_rate": 0.00012459792777027754, + "loss": 0.6755, + "step": 6490 + }, + { + "epoch": 0.6716263690845216, + "grad_norm": 0.813228189945221, + "learning_rate": 0.00012388944556255614, + "loss": 0.7517, + "step": 6500 + }, + { + "epoch": 0.6726596404215747, + "grad_norm": 1.0219289064407349, + "learning_rate": 0.0001231823192593625, + "loss": 0.647, + "step": 6510 + }, + { + "epoch": 0.6736929117586278, + "grad_norm": 1.7192949056625366, + "learning_rate": 0.00012247655646351597, + "loss": 0.7193, + "step": 6520 + }, + { + "epoch": 0.6747261830956809, + "grad_norm": 0.8978095054626465, + "learning_rate": 0.00012177216476317565, + "loss": 0.7004, + "step": 6530 + }, + { + "epoch": 0.675759454432734, + "grad_norm": 1.2175655364990234, + "learning_rate": 0.00012106915173175942, + "loss": 0.7798, + "step": 6540 + }, + { + "epoch": 0.6767927257697871, + "grad_norm": 0.7553932070732117, + "learning_rate": 0.00012036752492786171, + "loss": 0.7077, + "step": 6550 + }, + { + "epoch": 0.6778259971068402, + "grad_norm": 1.2669484615325928, + "learning_rate": 0.0001196672918951728, + "loss": 0.6207, + "step": 6560 + }, + { + "epoch": 0.6788592684438933, + "grad_norm": 1.0848954916000366, + "learning_rate": 0.00011896846016239752, + "loss": 0.7216, + "step": 6570 + }, + { + "epoch": 0.6798925397809464, + "grad_norm": 1.306950569152832, + "learning_rate": 0.00011827103724317426, + "loss": 0.8071, + "step": 6580 + }, + { + "epoch": 0.6809258111179995, + "grad_norm": 1.175700306892395, + "learning_rate": 0.00011757503063599436, + "loss": 0.7327, + "step": 6590 + }, + { + "epoch": 0.6819590824550527, + "grad_norm": 1.052412509918213, + "learning_rate": 0.00011688044782412124, + "loss": 0.7375, + "step": 6600 + }, + { + "epoch": 0.6829923537921058, + "grad_norm": 0.922924280166626, + "learning_rate": 0.00011618729627551037, + "loss": 0.7765, + "step": 6610 + }, + { + "epoch": 0.6840256251291589, + "grad_norm": 0.6624295711517334, + "learning_rate": 0.00011549558344272834, + "loss": 0.7365, + "step": 6620 + }, + { + "epoch": 0.685058896466212, + "grad_norm": 0.9047833681106567, + "learning_rate": 0.00011480531676287322, + "loss": 0.7549, + "step": 6630 + }, + { + "epoch": 0.6860921678032651, + "grad_norm": 0.8293854594230652, + "learning_rate": 0.00011411650365749454, + "loss": 0.7531, + "step": 6640 + }, + { + "epoch": 0.6871254391403182, + "grad_norm": 0.865106463432312, + "learning_rate": 0.00011342915153251332, + "loss": 0.6403, + "step": 6650 + }, + { + "epoch": 0.6881587104773713, + "grad_norm": 1.0484962463378906, + "learning_rate": 0.0001127432677781425, + "loss": 0.7709, + "step": 6660 + }, + { + "epoch": 0.6891919818144244, + "grad_norm": 0.9296818375587463, + "learning_rate": 0.00011205885976880753, + "loss": 0.6943, + "step": 6670 + }, + { + "epoch": 0.6902252531514775, + "grad_norm": 0.9692133069038391, + "learning_rate": 0.00011137593486306727, + "loss": 0.6684, + "step": 6680 + }, + { + "epoch": 0.6912585244885306, + "grad_norm": 1.2010059356689453, + "learning_rate": 0.00011069450040353448, + "loss": 0.8146, + "step": 6690 + }, + { + "epoch": 0.6922917958255838, + "grad_norm": 0.8989507555961609, + "learning_rate": 0.00011001456371679707, + "loss": 0.6553, + "step": 6700 + }, + { + "epoch": 0.6933250671626369, + "grad_norm": 1.166821837425232, + "learning_rate": 0.00010933613211333943, + "loss": 0.76, + "step": 6710 + }, + { + "epoch": 0.69435833849969, + "grad_norm": 0.8655950427055359, + "learning_rate": 0.00010865921288746362, + "loss": 0.7237, + "step": 6720 + }, + { + "epoch": 0.6953916098367431, + "grad_norm": 1.0120173692703247, + "learning_rate": 0.00010798381331721108, + "loss": 0.8003, + "step": 6730 + }, + { + "epoch": 0.6964248811737962, + "grad_norm": 1.4091392755508423, + "learning_rate": 0.0001073099406642844, + "loss": 0.8101, + "step": 6740 + }, + { + "epoch": 0.6974581525108493, + "grad_norm": 0.8245342969894409, + "learning_rate": 0.00010663760217396906, + "loss": 0.6408, + "step": 6750 + }, + { + "epoch": 0.6984914238479024, + "grad_norm": 0.7788679003715515, + "learning_rate": 0.00010596680507505596, + "loss": 0.7559, + "step": 6760 + }, + { + "epoch": 0.6995246951849555, + "grad_norm": 0.5751708745956421, + "learning_rate": 0.00010529755657976315, + "loss": 0.6268, + "step": 6770 + }, + { + "epoch": 0.7005579665220086, + "grad_norm": 0.9208303689956665, + "learning_rate": 0.00010462986388365837, + "loss": 0.7369, + "step": 6780 + }, + { + "epoch": 0.7015912378590617, + "grad_norm": 0.8211421966552734, + "learning_rate": 0.00010396373416558233, + "loss": 0.7131, + "step": 6790 + }, + { + "epoch": 0.7026245091961149, + "grad_norm": 0.6540753245353699, + "learning_rate": 0.00010329917458757068, + "loss": 0.7293, + "step": 6800 + }, + { + "epoch": 0.703657780533168, + "grad_norm": 1.3494755029678345, + "learning_rate": 0.0001026361922947775, + "loss": 0.6745, + "step": 6810 + }, + { + "epoch": 0.7046910518702211, + "grad_norm": 0.8039237856864929, + "learning_rate": 0.00010197479441539827, + "loss": 0.6993, + "step": 6820 + }, + { + "epoch": 0.7057243232072742, + "grad_norm": 0.8956775069236755, + "learning_rate": 0.00010131498806059352, + "loss": 0.7667, + "step": 6830 + }, + { + "epoch": 0.7067575945443273, + "grad_norm": 0.7461487650871277, + "learning_rate": 0.00010065678032441208, + "loss": 0.6487, + "step": 6840 + }, + { + "epoch": 0.7077908658813804, + "grad_norm": 1.2220842838287354, + "learning_rate": 0.00010000017828371458, + "loss": 0.6455, + "step": 6850 + }, + { + "epoch": 0.7088241372184335, + "grad_norm": 0.9173330068588257, + "learning_rate": 9.934518899809799e-05, + "loss": 0.6879, + "step": 6860 + }, + { + "epoch": 0.7098574085554866, + "grad_norm": 0.6598500609397888, + "learning_rate": 9.869181950981948e-05, + "loss": 0.7175, + "step": 6870 + }, + { + "epoch": 0.7108906798925397, + "grad_norm": 1.0425814390182495, + "learning_rate": 9.804007684372038e-05, + "loss": 0.7662, + "step": 6880 + }, + { + "epoch": 0.7119239512295928, + "grad_norm": 1.0084232091903687, + "learning_rate": 9.738996800715094e-05, + "loss": 0.7261, + "step": 6890 + }, + { + "epoch": 0.712957222566646, + "grad_norm": 0.9493532180786133, + "learning_rate": 9.674149998989523e-05, + "loss": 0.7575, + "step": 6900 + }, + { + "epoch": 0.7139904939036991, + "grad_norm": 0.6258692741394043, + "learning_rate": 9.609467976409525e-05, + "loss": 0.6136, + "step": 6910 + }, + { + "epoch": 0.7150237652407522, + "grad_norm": 0.6811854243278503, + "learning_rate": 9.544951428417667e-05, + "loss": 0.8021, + "step": 6920 + }, + { + "epoch": 0.7160570365778053, + "grad_norm": 0.8507707118988037, + "learning_rate": 9.480601048677371e-05, + "loss": 0.644, + "step": 6930 + }, + { + "epoch": 0.7170903079148584, + "grad_norm": 1.1403719186782837, + "learning_rate": 9.416417529065488e-05, + "loss": 0.7257, + "step": 6940 + }, + { + "epoch": 0.7181235792519115, + "grad_norm": 0.9672065377235413, + "learning_rate": 9.352401559664817e-05, + "loss": 0.6545, + "step": 6950 + }, + { + "epoch": 0.7191568505889646, + "grad_norm": 1.221876859664917, + "learning_rate": 9.288553828756702e-05, + "loss": 0.6231, + "step": 6960 + }, + { + "epoch": 0.7201901219260177, + "grad_norm": 0.8762854337692261, + "learning_rate": 9.224875022813645e-05, + "loss": 0.7172, + "step": 6970 + }, + { + "epoch": 0.7212233932630708, + "grad_norm": 0.811069667339325, + "learning_rate": 9.161365826491913e-05, + "loss": 0.6319, + "step": 6980 + }, + { + "epoch": 0.7222566646001239, + "grad_norm": 1.3847732543945312, + "learning_rate": 9.098026922624175e-05, + "loss": 0.8109, + "step": 6990 + }, + { + "epoch": 0.723289935937177, + "grad_norm": 1.142921805381775, + "learning_rate": 9.03485899221216e-05, + "loss": 0.7571, + "step": 7000 + }, + { + "epoch": 0.7243232072742302, + "grad_norm": 0.5236563086509705, + "learning_rate": 8.971862714419368e-05, + "loss": 0.6592, + "step": 7010 + }, + { + "epoch": 0.7253564786112833, + "grad_norm": 1.2576050758361816, + "learning_rate": 8.909038766563703e-05, + "loss": 0.7714, + "step": 7020 + }, + { + "epoch": 0.7263897499483364, + "grad_norm": 0.9121869802474976, + "learning_rate": 8.846387824110247e-05, + "loss": 0.8312, + "step": 7030 + }, + { + "epoch": 0.7274230212853895, + "grad_norm": 1.3254677057266235, + "learning_rate": 8.783910560663968e-05, + "loss": 0.7242, + "step": 7040 + }, + { + "epoch": 0.7284562926224426, + "grad_norm": 1.342081904411316, + "learning_rate": 8.721607647962496e-05, + "loss": 0.7367, + "step": 7050 + }, + { + "epoch": 0.7294895639594957, + "grad_norm": 1.3918049335479736, + "learning_rate": 8.659479755868882e-05, + "loss": 0.8162, + "step": 7060 + }, + { + "epoch": 0.7305228352965488, + "grad_norm": 0.9307500720024109, + "learning_rate": 8.597527552364415e-05, + "loss": 0.6592, + "step": 7070 + }, + { + "epoch": 0.7315561066336019, + "grad_norm": 0.958733081817627, + "learning_rate": 8.535751703541417e-05, + "loss": 0.7096, + "step": 7080 + }, + { + "epoch": 0.732589377970655, + "grad_norm": 0.6930942535400391, + "learning_rate": 8.474152873596122e-05, + "loss": 0.6513, + "step": 7090 + }, + { + "epoch": 0.7336226493077082, + "grad_norm": 1.3499219417572021, + "learning_rate": 8.412731724821482e-05, + "loss": 0.7495, + "step": 7100 + }, + { + "epoch": 0.7346559206447613, + "grad_norm": 0.9743627905845642, + "learning_rate": 8.351488917600077e-05, + "loss": 0.6989, + "step": 7110 + }, + { + "epoch": 0.7356891919818144, + "grad_norm": 0.8228883147239685, + "learning_rate": 8.290425110397018e-05, + "loss": 0.7556, + "step": 7120 + }, + { + "epoch": 0.7367224633188675, + "grad_norm": 0.9745365381240845, + "learning_rate": 8.229540959752851e-05, + "loss": 0.6756, + "step": 7130 + }, + { + "epoch": 0.7377557346559206, + "grad_norm": 0.9700762629508972, + "learning_rate": 8.168837120276512e-05, + "loss": 0.7298, + "step": 7140 + }, + { + "epoch": 0.7387890059929737, + "grad_norm": 1.2264478206634521, + "learning_rate": 8.108314244638267e-05, + "loss": 0.7307, + "step": 7150 + }, + { + "epoch": 0.7398222773300268, + "grad_norm": 1.1239992380142212, + "learning_rate": 8.047972983562748e-05, + "loss": 0.7043, + "step": 7160 + }, + { + "epoch": 0.7408555486670799, + "grad_norm": 0.8346306681632996, + "learning_rate": 7.987813985821882e-05, + "loss": 0.7194, + "step": 7170 + }, + { + "epoch": 0.741888820004133, + "grad_norm": 0.8321127891540527, + "learning_rate": 7.927837898227966e-05, + "loss": 0.7095, + "step": 7180 + }, + { + "epoch": 0.7429220913411861, + "grad_norm": 0.9774494767189026, + "learning_rate": 7.868045365626702e-05, + "loss": 0.7328, + "step": 7190 + }, + { + "epoch": 0.7439553626782393, + "grad_norm": 0.8818146586418152, + "learning_rate": 7.808437030890258e-05, + "loss": 0.6084, + "step": 7200 + }, + { + "epoch": 0.7449886340152924, + "grad_norm": 0.6830877661705017, + "learning_rate": 7.749013534910362e-05, + "loss": 0.6242, + "step": 7210 + }, + { + "epoch": 0.7460219053523455, + "grad_norm": 0.7373770475387573, + "learning_rate": 7.689775516591397e-05, + "loss": 0.6592, + "step": 7220 + }, + { + "epoch": 0.7470551766893986, + "grad_norm": 1.5116766691207886, + "learning_rate": 7.63072361284357e-05, + "loss": 0.7781, + "step": 7230 + }, + { + "epoch": 0.7480884480264517, + "grad_norm": 1.11928129196167, + "learning_rate": 7.571858458576017e-05, + "loss": 0.8028, + "step": 7240 + }, + { + "epoch": 0.7491217193635048, + "grad_norm": 0.5922040939331055, + "learning_rate": 7.513180686690002e-05, + "loss": 0.6601, + "step": 7250 + }, + { + "epoch": 0.750154990700558, + "grad_norm": 0.9148753881454468, + "learning_rate": 7.454690928072088e-05, + "loss": 0.7152, + "step": 7260 + }, + { + "epoch": 0.7511882620376111, + "grad_norm": 0.7024368047714233, + "learning_rate": 7.396389811587415e-05, + "loss": 0.6906, + "step": 7270 + }, + { + "epoch": 0.7522215333746642, + "grad_norm": 1.3108892440795898, + "learning_rate": 7.338277964072864e-05, + "loss": 0.7471, + "step": 7280 + }, + { + "epoch": 0.7532548047117174, + "grad_norm": 0.6525394320487976, + "learning_rate": 7.280356010330361e-05, + "loss": 0.7364, + "step": 7290 + }, + { + "epoch": 0.7542880760487705, + "grad_norm": 1.344499111175537, + "learning_rate": 7.222624573120137e-05, + "loss": 0.7023, + "step": 7300 + }, + { + "epoch": 0.7553213473858236, + "grad_norm": 0.7209261059761047, + "learning_rate": 7.165084273154074e-05, + "loss": 0.6748, + "step": 7310 + }, + { + "epoch": 0.7563546187228767, + "grad_norm": 1.4905532598495483, + "learning_rate": 7.10773572908898e-05, + "loss": 0.6376, + "step": 7320 + }, + { + "epoch": 0.7573878900599298, + "grad_norm": 0.4700772762298584, + "learning_rate": 7.050579557519938e-05, + "loss": 0.6694, + "step": 7330 + }, + { + "epoch": 0.7584211613969829, + "grad_norm": 0.6527320146560669, + "learning_rate": 6.993616372973739e-05, + "loss": 0.6764, + "step": 7340 + }, + { + "epoch": 0.759454432734036, + "grad_norm": 0.7010611295700073, + "learning_rate": 6.936846787902207e-05, + "loss": 0.658, + "step": 7350 + }, + { + "epoch": 0.7604877040710891, + "grad_norm": 1.1044212579727173, + "learning_rate": 6.880271412675645e-05, + "loss": 0.6569, + "step": 7360 + }, + { + "epoch": 0.7615209754081422, + "grad_norm": 0.8779295682907104, + "learning_rate": 6.823890855576261e-05, + "loss": 0.6382, + "step": 7370 + }, + { + "epoch": 0.7625542467451953, + "grad_norm": 0.9512449502944946, + "learning_rate": 6.76770572279167e-05, + "loss": 0.6453, + "step": 7380 + }, + { + "epoch": 0.7635875180822485, + "grad_norm": 0.782164990901947, + "learning_rate": 6.711716618408281e-05, + "loss": 0.6539, + "step": 7390 + }, + { + "epoch": 0.7646207894193016, + "grad_norm": 1.7972410917282104, + "learning_rate": 6.655924144404906e-05, + "loss": 0.6142, + "step": 7400 + }, + { + "epoch": 0.7656540607563547, + "grad_norm": 0.9399009346961975, + "learning_rate": 6.600328900646222e-05, + "loss": 0.7511, + "step": 7410 + }, + { + "epoch": 0.7666873320934078, + "grad_norm": 2.6008989810943604, + "learning_rate": 6.544931484876368e-05, + "loss": 0.5901, + "step": 7420 + }, + { + "epoch": 0.7677206034304609, + "grad_norm": 1.3178836107254028, + "learning_rate": 6.489732492712466e-05, + "loss": 0.6217, + "step": 7430 + }, + { + "epoch": 0.768753874767514, + "grad_norm": 1.1651490926742554, + "learning_rate": 6.434732517638254e-05, + "loss": 0.6472, + "step": 7440 + }, + { + "epoch": 0.7697871461045671, + "grad_norm": 1.072582721710205, + "learning_rate": 6.37993215099771e-05, + "loss": 0.6595, + "step": 7450 + }, + { + "epoch": 0.7708204174416202, + "grad_norm": 1.5162521600723267, + "learning_rate": 6.325331981988653e-05, + "loss": 0.7025, + "step": 7460 + }, + { + "epoch": 0.7718536887786733, + "grad_norm": 0.804498016834259, + "learning_rate": 6.27093259765645e-05, + "loss": 0.7168, + "step": 7470 + }, + { + "epoch": 0.7728869601157264, + "grad_norm": 1.0188984870910645, + "learning_rate": 6.216734582887678e-05, + "loss": 0.6158, + "step": 7480 + }, + { + "epoch": 0.7739202314527796, + "grad_norm": 0.9018194675445557, + "learning_rate": 6.162738520403871e-05, + "loss": 0.5752, + "step": 7490 + }, + { + "epoch": 0.7749535027898327, + "grad_norm": 0.6925551891326904, + "learning_rate": 6.108944990755202e-05, + "loss": 0.6934, + "step": 7500 + }, + { + "epoch": 0.7759867741268858, + "grad_norm": 1.3633153438568115, + "learning_rate": 6.0553545723142864e-05, + "loss": 0.6985, + "step": 7510 + }, + { + "epoch": 0.7770200454639389, + "grad_norm": 1.114071249961853, + "learning_rate": 6.001967841269937e-05, + "loss": 0.6031, + "step": 7520 + }, + { + "epoch": 0.778053316800992, + "grad_norm": 0.7959581613540649, + "learning_rate": 5.9487853716209794e-05, + "loss": 0.6645, + "step": 7530 + }, + { + "epoch": 0.7790865881380451, + "grad_norm": 0.8598949313163757, + "learning_rate": 5.89580773517008e-05, + "loss": 0.7458, + "step": 7540 + }, + { + "epoch": 0.7801198594750982, + "grad_norm": 1.4160740375518799, + "learning_rate": 5.843035501517596e-05, + "loss": 0.6635, + "step": 7550 + }, + { + "epoch": 0.7811531308121513, + "grad_norm": 1.0915048122406006, + "learning_rate": 5.790469238055465e-05, + "loss": 0.6052, + "step": 7560 + }, + { + "epoch": 0.7821864021492044, + "grad_norm": 0.8317732214927673, + "learning_rate": 5.738109509961076e-05, + "loss": 0.7007, + "step": 7570 + }, + { + "epoch": 0.7832196734862575, + "grad_norm": 0.9893842339515686, + "learning_rate": 5.685956880191218e-05, + "loss": 0.5915, + "step": 7580 + }, + { + "epoch": 0.7842529448233106, + "grad_norm": 1.2170053720474243, + "learning_rate": 5.634011909476008e-05, + "loss": 0.6921, + "step": 7590 + }, + { + "epoch": 0.7852862161603638, + "grad_norm": 1.261479377746582, + "learning_rate": 5.582275156312885e-05, + "loss": 0.6338, + "step": 7600 + }, + { + "epoch": 0.7863194874974169, + "grad_norm": 1.0150582790374756, + "learning_rate": 5.5307471769605875e-05, + "loss": 0.637, + "step": 7610 + }, + { + "epoch": 0.78735275883447, + "grad_norm": 1.148558497428894, + "learning_rate": 5.479428525433167e-05, + "loss": 0.6243, + "step": 7620 + }, + { + "epoch": 0.7883860301715231, + "grad_norm": 0.6112420558929443, + "learning_rate": 5.4283197534940684e-05, + "loss": 0.579, + "step": 7630 + }, + { + "epoch": 0.7894193015085762, + "grad_norm": 1.1648898124694824, + "learning_rate": 5.377421410650149e-05, + "loss": 0.6978, + "step": 7640 + }, + { + "epoch": 0.7904525728456293, + "grad_norm": 1.3245562314987183, + "learning_rate": 5.326734044145801e-05, + "loss": 0.5724, + "step": 7650 + }, + { + "epoch": 0.7914858441826824, + "grad_norm": 1.0554862022399902, + "learning_rate": 5.276258198957051e-05, + "loss": 0.7884, + "step": 7660 + }, + { + "epoch": 0.7925191155197355, + "grad_norm": 1.1452159881591797, + "learning_rate": 5.225994417785726e-05, + "loss": 0.7309, + "step": 7670 + }, + { + "epoch": 0.7935523868567886, + "grad_norm": 0.7298494577407837, + "learning_rate": 5.175943241053582e-05, + "loss": 0.6804, + "step": 7680 + }, + { + "epoch": 0.7945856581938417, + "grad_norm": 1.0335783958435059, + "learning_rate": 5.1261052068965306e-05, + "loss": 0.6306, + "step": 7690 + }, + { + "epoch": 0.7956189295308949, + "grad_norm": 0.6662817001342773, + "learning_rate": 5.0764808511588155e-05, + "loss": 0.6118, + "step": 7700 + }, + { + "epoch": 0.796652200867948, + "grad_norm": 0.8202494978904724, + "learning_rate": 5.027070707387296e-05, + "loss": 0.6764, + "step": 7710 + }, + { + "epoch": 0.7976854722050011, + "grad_norm": 1.5033270120620728, + "learning_rate": 4.977875306825672e-05, + "loss": 0.7725, + "step": 7720 + }, + { + "epoch": 0.7987187435420542, + "grad_norm": 0.7202991843223572, + "learning_rate": 4.928895178408782e-05, + "loss": 0.685, + "step": 7730 + }, + { + "epoch": 0.7997520148791073, + "grad_norm": 0.8215998411178589, + "learning_rate": 4.880130848756925e-05, + "loss": 0.6536, + "step": 7740 + }, + { + "epoch": 0.8007852862161604, + "grad_norm": 1.0507868528366089, + "learning_rate": 4.8315828421701986e-05, + "loss": 0.6238, + "step": 7750 + }, + { + "epoch": 0.8018185575532135, + "grad_norm": 1.0970897674560547, + "learning_rate": 4.783251680622847e-05, + "loss": 0.6302, + "step": 7760 + }, + { + "epoch": 0.8028518288902666, + "grad_norm": 1.2945756912231445, + "learning_rate": 4.7351378837576565e-05, + "loss": 0.5465, + "step": 7770 + }, + { + "epoch": 0.8038851002273197, + "grad_norm": 1.6131089925765991, + "learning_rate": 4.687241968880393e-05, + "loss": 0.6698, + "step": 7780 + }, + { + "epoch": 0.8049183715643728, + "grad_norm": 1.0525102615356445, + "learning_rate": 4.639564450954201e-05, + "loss": 0.6425, + "step": 7790 + }, + { + "epoch": 0.805951642901426, + "grad_norm": 1.0598100423812866, + "learning_rate": 4.5921058425940745e-05, + "loss": 0.5235, + "step": 7800 + }, + { + "epoch": 0.8069849142384791, + "grad_norm": 1.4403691291809082, + "learning_rate": 4.544866654061364e-05, + "loss": 0.7243, + "step": 7810 + }, + { + "epoch": 0.8080181855755322, + "grad_norm": 0.995022714138031, + "learning_rate": 4.497847393258292e-05, + "loss": 0.6382, + "step": 7820 + }, + { + "epoch": 0.8090514569125853, + "grad_norm": 1.2074267864227295, + "learning_rate": 4.451048565722468e-05, + "loss": 0.644, + "step": 7830 + }, + { + "epoch": 0.8100847282496384, + "grad_norm": 1.038830041885376, + "learning_rate": 4.404470674621463e-05, + "loss": 0.6955, + "step": 7840 + }, + { + "epoch": 0.8111179995866915, + "grad_norm": 1.028713345527649, + "learning_rate": 4.3581142207474225e-05, + "loss": 0.5929, + "step": 7850 + }, + { + "epoch": 0.8121512709237446, + "grad_norm": 1.214220404624939, + "learning_rate": 4.311979702511645e-05, + "loss": 0.6807, + "step": 7860 + }, + { + "epoch": 0.8131845422607977, + "grad_norm": 0.91230708360672, + "learning_rate": 4.266067615939234e-05, + "loss": 0.5622, + "step": 7870 + }, + { + "epoch": 0.8142178135978508, + "grad_norm": 0.7656465172767639, + "learning_rate": 4.220378454663784e-05, + "loss": 0.7191, + "step": 7880 + }, + { + "epoch": 0.815251084934904, + "grad_norm": 0.9677980542182922, + "learning_rate": 4.1749127099220684e-05, + "loss": 0.6268, + "step": 7890 + }, + { + "epoch": 0.8162843562719571, + "grad_norm": 0.8608049750328064, + "learning_rate": 4.129670870548738e-05, + "loss": 0.608, + "step": 7900 + }, + { + "epoch": 0.8173176276090102, + "grad_norm": 1.3923230171203613, + "learning_rate": 4.084653422971077e-05, + "loss": 0.6626, + "step": 7910 + }, + { + "epoch": 0.8183508989460633, + "grad_norm": 1.0703678131103516, + "learning_rate": 4.039860851203775e-05, + "loss": 0.5827, + "step": 7920 + }, + { + "epoch": 0.8193841702831164, + "grad_norm": 0.9771378040313721, + "learning_rate": 3.9952936368437275e-05, + "loss": 0.7703, + "step": 7930 + }, + { + "epoch": 0.8204174416201695, + "grad_norm": 1.0725332498550415, + "learning_rate": 3.950952259064841e-05, + "loss": 0.6425, + "step": 7940 + }, + { + "epoch": 0.8214507129572226, + "grad_norm": 1.1629067659378052, + "learning_rate": 3.9068371946128945e-05, + "loss": 0.6658, + "step": 7950 + }, + { + "epoch": 0.8224839842942757, + "grad_norm": 0.9839202165603638, + "learning_rate": 3.862948917800424e-05, + "loss": 0.5883, + "step": 7960 + }, + { + "epoch": 0.8235172556313288, + "grad_norm": 0.8677191138267517, + "learning_rate": 3.8192879005015974e-05, + "loss": 0.6664, + "step": 7970 + }, + { + "epoch": 0.8245505269683819, + "grad_norm": 0.8415852189064026, + "learning_rate": 3.775854612147153e-05, + "loss": 0.5629, + "step": 7980 + }, + { + "epoch": 0.825583798305435, + "grad_norm": 0.7151034474372864, + "learning_rate": 3.7326495197193555e-05, + "loss": 0.5575, + "step": 7990 + }, + { + "epoch": 0.8266170696424882, + "grad_norm": 1.2307301759719849, + "learning_rate": 3.689673087746975e-05, + "loss": 0.6356, + "step": 8000 + }, + { + "epoch": 0.8276503409795413, + "grad_norm": 2.110441207885742, + "learning_rate": 3.646925778300286e-05, + "loss": 0.7123, + "step": 8010 + }, + { + "epoch": 0.8286836123165944, + "grad_norm": 0.47642096877098083, + "learning_rate": 3.604408050986105e-05, + "loss": 0.6165, + "step": 8020 + }, + { + "epoch": 0.8297168836536475, + "grad_norm": 0.69282066822052, + "learning_rate": 3.5621203629428386e-05, + "loss": 0.6196, + "step": 8030 + }, + { + "epoch": 0.8307501549907006, + "grad_norm": 1.5054296255111694, + "learning_rate": 3.520063168835605e-05, + "loss": 0.749, + "step": 8040 + }, + { + "epoch": 0.8317834263277537, + "grad_norm": 1.0589144229888916, + "learning_rate": 3.478236920851283e-05, + "loss": 0.6056, + "step": 8050 + }, + { + "epoch": 0.8328166976648068, + "grad_norm": 0.8430522680282593, + "learning_rate": 3.4366420686937044e-05, + "loss": 0.6502, + "step": 8060 + }, + { + "epoch": 0.8338499690018599, + "grad_norm": 0.8304775357246399, + "learning_rate": 3.395279059578798e-05, + "loss": 0.6363, + "step": 8070 + }, + { + "epoch": 0.834883240338913, + "grad_norm": 0.5285193920135498, + "learning_rate": 3.3541483382297734e-05, + "loss": 0.751, + "step": 8080 + }, + { + "epoch": 0.8359165116759661, + "grad_norm": 0.9843568205833435, + "learning_rate": 3.313250346872362e-05, + "loss": 0.5978, + "step": 8090 + }, + { + "epoch": 0.8369497830130193, + "grad_norm": 1.0727510452270508, + "learning_rate": 3.272585525230032e-05, + "loss": 0.528, + "step": 8100 + }, + { + "epoch": 0.8379830543500724, + "grad_norm": 0.9402909874916077, + "learning_rate": 3.232154310519298e-05, + "loss": 0.5649, + "step": 8110 + }, + { + "epoch": 0.8390163256871255, + "grad_norm": 0.6825330257415771, + "learning_rate": 3.1919571374449894e-05, + "loss": 0.6619, + "step": 8120 + }, + { + "epoch": 0.8400495970241786, + "grad_norm": 1.605404019355774, + "learning_rate": 3.151994438195582e-05, + "loss": 0.7536, + "step": 8130 + }, + { + "epoch": 0.8410828683612317, + "grad_norm": 0.9325007796287537, + "learning_rate": 3.112266642438566e-05, + "loss": 0.7271, + "step": 8140 + }, + { + "epoch": 0.8421161396982848, + "grad_norm": 1.083452820777893, + "learning_rate": 3.072774177315807e-05, + "loss": 0.6664, + "step": 8150 + }, + { + "epoch": 0.8431494110353379, + "grad_norm": 0.6602939963340759, + "learning_rate": 3.0335174674389732e-05, + "loss": 0.5289, + "step": 8160 + }, + { + "epoch": 0.844182682372391, + "grad_norm": 1.497051477432251, + "learning_rate": 2.9944969348849493e-05, + "loss": 0.5557, + "step": 8170 + }, + { + "epoch": 0.8452159537094441, + "grad_norm": 0.7010759115219116, + "learning_rate": 2.9557129991913235e-05, + "loss": 0.5932, + "step": 8180 + }, + { + "epoch": 0.8462492250464972, + "grad_norm": 0.93887859582901, + "learning_rate": 2.917166077351846e-05, + "loss": 0.5421, + "step": 8190 + }, + { + "epoch": 0.8472824963835504, + "grad_norm": 1.242521047592163, + "learning_rate": 2.8788565838119757e-05, + "loss": 0.5551, + "step": 8200 + }, + { + "epoch": 0.8483157677206035, + "grad_norm": 1.1588674783706665, + "learning_rate": 2.840784930464399e-05, + "loss": 0.5647, + "step": 8210 + }, + { + "epoch": 0.8493490390576566, + "grad_norm": 1.381052851676941, + "learning_rate": 2.8029515266446232e-05, + "loss": 0.6187, + "step": 8220 + }, + { + "epoch": 0.8503823103947097, + "grad_norm": 1.1169166564941406, + "learning_rate": 2.765356779126557e-05, + "loss": 0.6038, + "step": 8230 + }, + { + "epoch": 0.8514155817317628, + "grad_norm": 1.2974213361740112, + "learning_rate": 2.7280010921181487e-05, + "loss": 0.7017, + "step": 8240 + }, + { + "epoch": 0.8524488530688159, + "grad_norm": 0.9372439384460449, + "learning_rate": 2.690884867257032e-05, + "loss": 0.5887, + "step": 8250 + }, + { + "epoch": 0.853482124405869, + "grad_norm": 0.8471134305000305, + "learning_rate": 2.6540085036062227e-05, + "loss": 0.5159, + "step": 8260 + }, + { + "epoch": 0.8545153957429221, + "grad_norm": 1.077082633972168, + "learning_rate": 2.617372397649814e-05, + "loss": 0.6389, + "step": 8270 + }, + { + "epoch": 0.8555486670799752, + "grad_norm": 1.1442515850067139, + "learning_rate": 2.5809769432886986e-05, + "loss": 0.6233, + "step": 8280 + }, + { + "epoch": 0.8565819384170283, + "grad_norm": 1.59578537940979, + "learning_rate": 2.5448225318363828e-05, + "loss": 0.6336, + "step": 8290 + }, + { + "epoch": 0.8576152097540815, + "grad_norm": 0.7789118885993958, + "learning_rate": 2.5089095520147266e-05, + "loss": 0.6106, + "step": 8300 + }, + { + "epoch": 0.8586484810911346, + "grad_norm": 0.8877026438713074, + "learning_rate": 2.4732383899497972e-05, + "loss": 0.6226, + "step": 8310 + }, + { + "epoch": 0.8596817524281877, + "grad_norm": 0.9339987635612488, + "learning_rate": 2.4378094291676978e-05, + "loss": 0.5998, + "step": 8320 + }, + { + "epoch": 0.8607150237652408, + "grad_norm": 0.45565560460090637, + "learning_rate": 2.4026230505904633e-05, + "loss": 0.5463, + "step": 8330 + }, + { + "epoch": 0.8617482951022939, + "grad_norm": 0.7657277584075928, + "learning_rate": 2.367679632531955e-05, + "loss": 0.6507, + "step": 8340 + }, + { + "epoch": 0.862781566439347, + "grad_norm": 1.4094293117523193, + "learning_rate": 2.3329795506937728e-05, + "loss": 0.5541, + "step": 8350 + }, + { + "epoch": 0.8638148377764001, + "grad_norm": 0.5916699171066284, + "learning_rate": 2.2985231781612466e-05, + "loss": 0.6416, + "step": 8360 + }, + { + "epoch": 0.8648481091134532, + "grad_norm": 0.7877978086471558, + "learning_rate": 2.2643108853994317e-05, + "loss": 0.6413, + "step": 8370 + }, + { + "epoch": 0.8658813804505063, + "grad_norm": 1.3543034791946411, + "learning_rate": 2.2303430402490805e-05, + "loss": 0.6255, + "step": 8380 + }, + { + "epoch": 0.8669146517875594, + "grad_norm": 1.2339602708816528, + "learning_rate": 2.1966200079227257e-05, + "loss": 0.6086, + "step": 8390 + }, + { + "epoch": 0.8679479231246126, + "grad_norm": 1.563103199005127, + "learning_rate": 2.1631421510007575e-05, + "loss": 0.6084, + "step": 8400 + }, + { + "epoch": 0.8689811944616657, + "grad_norm": 1.1265037059783936, + "learning_rate": 2.129909829427479e-05, + "loss": 0.5604, + "step": 8410 + }, + { + "epoch": 0.8700144657987188, + "grad_norm": 1.4551678895950317, + "learning_rate": 2.0969234005072916e-05, + "loss": 0.6835, + "step": 8420 + }, + { + "epoch": 0.8710477371357719, + "grad_norm": 0.9663675427436829, + "learning_rate": 2.064183218900817e-05, + "loss": 0.6, + "step": 8430 + }, + { + "epoch": 0.872081008472825, + "grad_norm": 1.2805395126342773, + "learning_rate": 2.0316896366211118e-05, + "loss": 0.6471, + "step": 8440 + }, + { + "epoch": 0.8731142798098781, + "grad_norm": 0.9442639946937561, + "learning_rate": 1.9994430030298496e-05, + "loss": 0.6104, + "step": 8450 + }, + { + "epoch": 0.8741475511469312, + "grad_norm": 1.5761253833770752, + "learning_rate": 1.967443664833596e-05, + "loss": 0.6629, + "step": 8460 + }, + { + "epoch": 0.8751808224839843, + "grad_norm": 1.1689879894256592, + "learning_rate": 1.9356919660800553e-05, + "loss": 0.6161, + "step": 8470 + }, + { + "epoch": 0.8762140938210374, + "grad_norm": 1.1315240859985352, + "learning_rate": 1.9041882481543955e-05, + "loss": 0.574, + "step": 8480 + }, + { + "epoch": 0.8772473651580905, + "grad_norm": 1.0627079010009766, + "learning_rate": 1.8729328497755578e-05, + "loss": 0.6021, + "step": 8490 + }, + { + "epoch": 0.8782806364951437, + "grad_norm": 0.9222522377967834, + "learning_rate": 1.8419261069926197e-05, + "loss": 0.5026, + "step": 8500 + }, + { + "epoch": 0.8793139078321968, + "grad_norm": 0.92143315076828, + "learning_rate": 1.8111683531812002e-05, + "loss": 0.5853, + "step": 8510 + }, + { + "epoch": 0.8803471791692499, + "grad_norm": 1.042581558227539, + "learning_rate": 1.7806599190398455e-05, + "loss": 0.5239, + "step": 8520 + }, + { + "epoch": 0.881380450506303, + "grad_norm": 1.405364751815796, + "learning_rate": 1.750401132586496e-05, + "loss": 0.571, + "step": 8530 + }, + { + "epoch": 0.8824137218433561, + "grad_norm": 1.2399895191192627, + "learning_rate": 1.720392319154948e-05, + "loss": 0.5704, + "step": 8540 + }, + { + "epoch": 0.8834469931804092, + "grad_norm": 0.9870766401290894, + "learning_rate": 1.6906338013913585e-05, + "loss": 0.5944, + "step": 8550 + }, + { + "epoch": 0.8844802645174623, + "grad_norm": 0.8951932787895203, + "learning_rate": 1.6611258992507817e-05, + "loss": 0.5861, + "step": 8560 + }, + { + "epoch": 0.8855135358545154, + "grad_norm": 1.8214749097824097, + "learning_rate": 1.6318689299937213e-05, + "loss": 0.6643, + "step": 8570 + }, + { + "epoch": 0.8865468071915685, + "grad_norm": 0.8029395341873169, + "learning_rate": 1.6028632081827173e-05, + "loss": 0.5645, + "step": 8580 + }, + { + "epoch": 0.8875800785286216, + "grad_norm": 0.43516650795936584, + "learning_rate": 1.574109045678987e-05, + "loss": 0.5996, + "step": 8590 + }, + { + "epoch": 0.8886133498656748, + "grad_norm": 1.673608422279358, + "learning_rate": 1.545606751639034e-05, + "loss": 0.6781, + "step": 8600 + }, + { + "epoch": 0.8896466212027279, + "grad_norm": 1.0479223728179932, + "learning_rate": 1.5173566325113575e-05, + "loss": 0.5522, + "step": 8610 + }, + { + "epoch": 0.890679892539781, + "grad_norm": 1.2102528810501099, + "learning_rate": 1.4893589920331352e-05, + "loss": 0.5926, + "step": 8620 + }, + { + "epoch": 0.8917131638768341, + "grad_norm": 1.0137587785720825, + "learning_rate": 1.4616141312269737e-05, + "loss": 0.5393, + "step": 8630 + }, + { + "epoch": 0.8927464352138872, + "grad_norm": 2.0007264614105225, + "learning_rate": 1.4341223483976624e-05, + "loss": 0.6629, + "step": 8640 + }, + { + "epoch": 0.8937797065509403, + "grad_norm": 0.966791570186615, + "learning_rate": 1.406883939128964e-05, + "loss": 0.6255, + "step": 8650 + }, + { + "epoch": 0.8948129778879934, + "grad_norm": 1.436010479927063, + "learning_rate": 1.3798991962804541e-05, + "loss": 0.6226, + "step": 8660 + }, + { + "epoch": 0.8958462492250465, + "grad_norm": 1.060235857963562, + "learning_rate": 1.3531684099843483e-05, + "loss": 0.6685, + "step": 8670 + }, + { + "epoch": 0.8968795205620996, + "grad_norm": 0.578377902507782, + "learning_rate": 1.3266918676423962e-05, + "loss": 0.5681, + "step": 8680 + }, + { + "epoch": 0.8979127918991527, + "grad_norm": 0.5564404726028442, + "learning_rate": 1.300469853922795e-05, + "loss": 0.587, + "step": 8690 + }, + { + "epoch": 0.8989460632362059, + "grad_norm": 0.9987608194351196, + "learning_rate": 1.2745026507571145e-05, + "loss": 0.65, + "step": 8700 + }, + { + "epoch": 0.899979334573259, + "grad_norm": 1.4747377634048462, + "learning_rate": 1.2487905373372799e-05, + "loss": 0.6024, + "step": 8710 + }, + { + "epoch": 0.9010126059103121, + "grad_norm": 0.7746207118034363, + "learning_rate": 1.223333790112563e-05, + "loss": 0.6037, + "step": 8720 + }, + { + "epoch": 0.9020458772473652, + "grad_norm": 1.0953818559646606, + "learning_rate": 1.1981326827866124e-05, + "loss": 0.6469, + "step": 8730 + }, + { + "epoch": 0.9030791485844183, + "grad_norm": 1.1232373714447021, + "learning_rate": 1.1731874863145143e-05, + "loss": 0.591, + "step": 8740 + }, + { + "epoch": 0.9041124199214714, + "grad_norm": 2.062878131866455, + "learning_rate": 1.148498468899864e-05, + "loss": 0.6622, + "step": 8750 + }, + { + "epoch": 0.9051456912585245, + "grad_norm": 1.1355712413787842, + "learning_rate": 1.1240658959918965e-05, + "loss": 0.762, + "step": 8760 + }, + { + "epoch": 0.9061789625955776, + "grad_norm": 1.6199957132339478, + "learning_rate": 1.0998900302826382e-05, + "loss": 0.6976, + "step": 8770 + }, + { + "epoch": 0.9072122339326307, + "grad_norm": 1.111836552619934, + "learning_rate": 1.0759711317040649e-05, + "loss": 0.5116, + "step": 8780 + }, + { + "epoch": 0.9082455052696838, + "grad_norm": 1.650025725364685, + "learning_rate": 1.0523094574253101e-05, + "loss": 0.5384, + "step": 8790 + }, + { + "epoch": 0.909278776606737, + "grad_norm": 0.9917317628860474, + "learning_rate": 1.0289052618499218e-05, + "loss": 0.5526, + "step": 8800 + }, + { + "epoch": 0.9103120479437901, + "grad_norm": 0.618635356426239, + "learning_rate": 1.005758796613096e-05, + "loss": 0.6995, + "step": 8810 + }, + { + "epoch": 0.9113453192808432, + "grad_norm": 1.289839506149292, + "learning_rate": 9.828703105789983e-06, + "loss": 0.6453, + "step": 8820 + }, + { + "epoch": 0.9123785906178963, + "grad_norm": 1.1484878063201904, + "learning_rate": 9.602400498380542e-06, + "loss": 0.5949, + "step": 8830 + }, + { + "epoch": 0.9134118619549494, + "grad_norm": 1.1618013381958008, + "learning_rate": 9.378682577043524e-06, + "loss": 0.633, + "step": 8840 + }, + { + "epoch": 0.9144451332920025, + "grad_norm": 0.8447564840316772, + "learning_rate": 9.157551747129844e-06, + "loss": 0.6921, + "step": 8850 + }, + { + "epoch": 0.9154784046290556, + "grad_norm": 2.2064759731292725, + "learning_rate": 8.939010386174783e-06, + "loss": 0.5906, + "step": 8860 + }, + { + "epoch": 0.9165116759661087, + "grad_norm": 1.3931576013565063, + "learning_rate": 8.723060843872393e-06, + "loss": 0.6567, + "step": 8870 + }, + { + "epoch": 0.9175449473031618, + "grad_norm": 0.8289423584938049, + "learning_rate": 8.50970544205032e-06, + "loss": 0.6287, + "step": 8880 + }, + { + "epoch": 0.9185782186402149, + "grad_norm": 0.911371111869812, + "learning_rate": 8.298946474644575e-06, + "loss": 0.541, + "step": 8890 + }, + { + "epoch": 0.919611489977268, + "grad_norm": 1.081002950668335, + "learning_rate": 8.090786207675171e-06, + "loss": 0.6585, + "step": 8900 + }, + { + "epoch": 0.9206447613143212, + "grad_norm": 1.1589564085006714, + "learning_rate": 7.885226879221691e-06, + "loss": 0.644, + "step": 8910 + }, + { + "epoch": 0.9216780326513743, + "grad_norm": 0.6146034002304077, + "learning_rate": 7.682270699399057e-06, + "loss": 0.525, + "step": 8920 + }, + { + "epoch": 0.9227113039884274, + "grad_norm": 0.8697605729103088, + "learning_rate": 7.481919850333946e-06, + "loss": 0.5737, + "step": 8930 + }, + { + "epoch": 0.9237445753254805, + "grad_norm": 1.1086236238479614, + "learning_rate": 7.284176486141214e-06, + "loss": 0.5475, + "step": 8940 + }, + { + "epoch": 0.9247778466625336, + "grad_norm": 1.8016564846038818, + "learning_rate": 7.0890427329008964e-06, + "loss": 0.6598, + "step": 8950 + }, + { + "epoch": 0.9258111179995867, + "grad_norm": 1.1818935871124268, + "learning_rate": 6.896520688635111e-06, + "loss": 0.634, + "step": 8960 + }, + { + "epoch": 0.9268443893366398, + "grad_norm": 1.2800990343093872, + "learning_rate": 6.706612423285685e-06, + "loss": 0.6006, + "step": 8970 + }, + { + "epoch": 0.9278776606736929, + "grad_norm": 1.0055065155029297, + "learning_rate": 6.519319978691845e-06, + "loss": 0.5678, + "step": 8980 + }, + { + "epoch": 0.928910932010746, + "grad_norm": 2.050903081893921, + "learning_rate": 6.334645368568315e-06, + "loss": 0.6205, + "step": 8990 + }, + { + "epoch": 0.9299442033477991, + "grad_norm": 1.3951302766799927, + "learning_rate": 6.152590578483497e-06, + "loss": 0.5148, + "step": 9000 + }, + { + "epoch": 0.9309774746848523, + "grad_norm": 2.197643995285034, + "learning_rate": 5.973157565838327e-06, + "loss": 0.6931, + "step": 9010 + }, + { + "epoch": 0.9320107460219054, + "grad_norm": 1.161935567855835, + "learning_rate": 5.796348259845096e-06, + "loss": 0.5684, + "step": 9020 + }, + { + "epoch": 0.9330440173589585, + "grad_norm": 1.7656304836273193, + "learning_rate": 5.622164561506826e-06, + "loss": 0.6114, + "step": 9030 + }, + { + "epoch": 0.9340772886960116, + "grad_norm": 0.9781146049499512, + "learning_rate": 5.450608343596647e-06, + "loss": 0.5743, + "step": 9040 + }, + { + "epoch": 0.9351105600330647, + "grad_norm": 0.7141128182411194, + "learning_rate": 5.281681450637843e-06, + "loss": 0.5756, + "step": 9050 + }, + { + "epoch": 0.9361438313701178, + "grad_norm": 1.8938722610473633, + "learning_rate": 5.115385698883979e-06, + "loss": 0.5679, + "step": 9060 + }, + { + "epoch": 0.9371771027071709, + "grad_norm": 1.8188812732696533, + "learning_rate": 4.951722876299275e-06, + "loss": 0.744, + "step": 9070 + }, + { + "epoch": 0.938210374044224, + "grad_norm": 0.8902860879898071, + "learning_rate": 4.790694742539487e-06, + "loss": 0.6274, + "step": 9080 + }, + { + "epoch": 0.9392436453812771, + "grad_norm": 2.0708553791046143, + "learning_rate": 4.632303028932943e-06, + "loss": 0.6173, + "step": 9090 + }, + { + "epoch": 0.9402769167183302, + "grad_norm": 1.710466742515564, + "learning_rate": 4.476549438461958e-06, + "loss": 0.6267, + "step": 9100 + }, + { + "epoch": 0.9413101880553834, + "grad_norm": 0.8895155787467957, + "learning_rate": 4.323435645744445e-06, + "loss": 0.7292, + "step": 9110 + }, + { + "epoch": 0.9423434593924365, + "grad_norm": 1.9920309782028198, + "learning_rate": 4.172963297015997e-06, + "loss": 0.6778, + "step": 9120 + }, + { + "epoch": 0.9433767307294896, + "grad_norm": 1.4172946214675903, + "learning_rate": 4.025134010112258e-06, + "loss": 0.5764, + "step": 9130 + }, + { + "epoch": 0.9444100020665427, + "grad_norm": 0.9763116836547852, + "learning_rate": 3.879949374451269e-06, + "loss": 0.5936, + "step": 9140 + }, + { + "epoch": 0.9454432734035958, + "grad_norm": 0.7206150889396667, + "learning_rate": 3.737410951016623e-06, + "loss": 0.6432, + "step": 9150 + }, + { + "epoch": 0.9464765447406489, + "grad_norm": 1.7434405088424683, + "learning_rate": 3.5975202723405874e-06, + "loss": 0.6962, + "step": 9160 + }, + { + "epoch": 0.947509816077702, + "grad_norm": 1.5172473192214966, + "learning_rate": 3.4602788424876464e-06, + "loss": 0.6048, + "step": 9170 + }, + { + "epoch": 0.9485430874147551, + "grad_norm": 1.1150609254837036, + "learning_rate": 3.3256881370383183e-06, + "loss": 0.5865, + "step": 9180 + }, + { + "epoch": 0.9495763587518082, + "grad_norm": 0.8272204995155334, + "learning_rate": 3.1937496030732536e-06, + "loss": 0.7075, + "step": 9190 + }, + { + "epoch": 0.9506096300888613, + "grad_norm": 0.9530496597290039, + "learning_rate": 3.0644646591577174e-06, + "loss": 0.5166, + "step": 9200 + }, + { + "epoch": 0.9516429014259145, + "grad_norm": 0.7508150935173035, + "learning_rate": 2.9378346953264633e-06, + "loss": 0.5569, + "step": 9210 + }, + { + "epoch": 0.9526761727629676, + "grad_norm": 1.5538454055786133, + "learning_rate": 2.8138610730684686e-06, + "loss": 0.6117, + "step": 9220 + }, + { + "epoch": 0.9537094441000207, + "grad_norm": 1.6959922313690186, + "learning_rate": 2.692545125312612e-06, + "loss": 0.5843, + "step": 9230 + }, + { + "epoch": 0.9547427154370738, + "grad_norm": 1.9940367937088013, + "learning_rate": 2.573888156413212e-06, + "loss": 0.6021, + "step": 9240 + }, + { + "epoch": 0.9557759867741269, + "grad_norm": 1.3337304592132568, + "learning_rate": 2.4578914421359288e-06, + "loss": 0.525, + "step": 9250 + }, + { + "epoch": 0.95680925811118, + "grad_norm": 0.8987991213798523, + "learning_rate": 2.344556229644218e-06, + "loss": 0.6227, + "step": 9260 + }, + { + "epoch": 0.9578425294482331, + "grad_norm": 0.8351725339889526, + "learning_rate": 2.233883737485731e-06, + "loss": 0.5019, + "step": 9270 + }, + { + "epoch": 0.9588758007852862, + "grad_norm": 0.8255094289779663, + "learning_rate": 2.1258751555794654e-06, + "loss": 0.6226, + "step": 9280 + }, + { + "epoch": 0.9599090721223393, + "grad_norm": 0.882256269454956, + "learning_rate": 2.020531645202689e-06, + "loss": 0.4691, + "step": 9290 + }, + { + "epoch": 0.9609423434593924, + "grad_norm": 1.3614681959152222, + "learning_rate": 1.9178543389786475e-06, + "loss": 0.6947, + "step": 9300 + }, + { + "epoch": 0.9619756147964456, + "grad_norm": 1.0351027250289917, + "learning_rate": 1.8178443408642386e-06, + "loss": 0.5705, + "step": 9310 + }, + { + "epoch": 0.9630088861334987, + "grad_norm": 1.4347681999206543, + "learning_rate": 1.7205027261383565e-06, + "loss": 0.659, + "step": 9320 + }, + { + "epoch": 0.9640421574705518, + "grad_norm": 0.7792657017707825, + "learning_rate": 1.625830541390122e-06, + "loss": 0.5753, + "step": 9330 + }, + { + "epoch": 0.9650754288076049, + "grad_norm": 2.364046812057495, + "learning_rate": 1.5338288045076697e-06, + "loss": 0.626, + "step": 9340 + }, + { + "epoch": 0.966108700144658, + "grad_norm": 0.6518005728721619, + "learning_rate": 1.4444985046673798e-06, + "loss": 0.5726, + "step": 9350 + }, + { + "epoch": 0.9671419714817111, + "grad_norm": 0.8066127896308899, + "learning_rate": 1.3578406023229962e-06, + "loss": 0.6068, + "step": 9360 + }, + { + "epoch": 0.9681752428187642, + "grad_norm": 0.918552815914154, + "learning_rate": 1.2738560291954415e-06, + "loss": 0.5854, + "step": 9370 + }, + { + "epoch": 0.9692085141558173, + "grad_norm": 0.840164840221405, + "learning_rate": 1.1925456882627417e-06, + "loss": 0.5582, + "step": 9380 + }, + { + "epoch": 0.9702417854928704, + "grad_norm": 1.3730981349945068, + "learning_rate": 1.113910453750394e-06, + "loss": 0.6132, + "step": 9390 + }, + { + "epoch": 0.9712750568299235, + "grad_norm": 0.9575563669204712, + "learning_rate": 1.0379511711219313e-06, + "loss": 0.552, + "step": 9400 + }, + { + "epoch": 0.9723083281669767, + "grad_norm": 1.423699975013733, + "learning_rate": 9.646686570697061e-07, + "loss": 0.6279, + "step": 9410 + }, + { + "epoch": 0.9733415995040298, + "grad_norm": 1.0341185331344604, + "learning_rate": 8.9406369950637e-07, + "loss": 0.6182, + "step": 9420 + }, + { + "epoch": 0.9743748708410829, + "grad_norm": 0.9466649889945984, + "learning_rate": 8.261370575561866e-07, + "loss": 0.6104, + "step": 9430 + }, + { + "epoch": 0.975408142178136, + "grad_norm": 1.4443756341934204, + "learning_rate": 7.608894615468709e-07, + "loss": 0.6099, + "step": 9440 + }, + { + "epoch": 0.9764414135151891, + "grad_norm": 1.057411551475525, + "learning_rate": 6.983216130019288e-07, + "loss": 0.621, + "step": 9450 + }, + { + "epoch": 0.9774746848522422, + "grad_norm": 1.0595217943191528, + "learning_rate": 6.384341846329134e-07, + "loss": 0.6057, + "step": 9460 + }, + { + "epoch": 0.9785079561892953, + "grad_norm": 1.049188494682312, + "learning_rate": 5.812278203322918e-07, + "loss": 0.5873, + "step": 9470 + }, + { + "epoch": 0.9795412275263484, + "grad_norm": 1.212692379951477, + "learning_rate": 5.267031351664786e-07, + "loss": 0.6586, + "step": 9480 + }, + { + "epoch": 0.9805744988634015, + "grad_norm": 0.6824125647544861, + "learning_rate": 4.7486071536925745e-07, + "loss": 0.5165, + "step": 9490 + }, + { + "epoch": 0.9816077702004546, + "grad_norm": 0.6535896062850952, + "learning_rate": 4.257011183354809e-07, + "loss": 0.568, + "step": 9500 + }, + { + "epoch": 0.9826410415375078, + "grad_norm": 1.9366480112075806, + "learning_rate": 3.792248726150471e-07, + "loss": 0.5708, + "step": 9510 + }, + { + "epoch": 0.9836743128745609, + "grad_norm": 0.8946526646614075, + "learning_rate": 3.354324779071827e-07, + "loss": 0.499, + "step": 9520 + }, + { + "epoch": 0.984707584211614, + "grad_norm": 1.2145295143127441, + "learning_rate": 2.9432440505522406e-07, + "loss": 0.6079, + "step": 9530 + }, + { + "epoch": 0.9857408555486671, + "grad_norm": 0.948176383972168, + "learning_rate": 2.559010960413444e-07, + "loss": 0.5659, + "step": 9540 + }, + { + "epoch": 0.9867741268857202, + "grad_norm": 0.8348559141159058, + "learning_rate": 2.201629639819458e-07, + "loss": 0.6056, + "step": 9550 + }, + { + "epoch": 0.9878073982227733, + "grad_norm": 1.4996442794799805, + "learning_rate": 1.87110393123191e-07, + "loss": 0.521, + "step": 9560 + }, + { + "epoch": 0.9888406695598264, + "grad_norm": 0.999030590057373, + "learning_rate": 1.567437388368398e-07, + "loss": 0.6173, + "step": 9570 + }, + { + "epoch": 0.9898739408968795, + "grad_norm": 2.4354379177093506, + "learning_rate": 1.2906332761647433e-07, + "loss": 0.6978, + "step": 9580 + }, + { + "epoch": 0.9909072122339326, + "grad_norm": 1.067911148071289, + "learning_rate": 1.0406945707391868e-07, + "loss": 0.5733, + "step": 9590 + }, + { + "epoch": 0.9919404835709857, + "grad_norm": 0.8200865387916565, + "learning_rate": 8.176239593610246e-08, + "loss": 0.602, + "step": 9600 + }, + { + "epoch": 0.9929737549080389, + "grad_norm": 0.7436734437942505, + "learning_rate": 6.214238404214645e-08, + "loss": 0.6507, + "step": 9610 + }, + { + "epoch": 0.994007026245092, + "grad_norm": 1.0331168174743652, + "learning_rate": 4.520963234083686e-08, + "loss": 0.6224, + "step": 9620 + }, + { + "epoch": 0.9950402975821451, + "grad_norm": 1.3714491128921509, + "learning_rate": 3.09643228882106e-08, + "loss": 0.6278, + "step": 9630 + }, + { + "epoch": 0.9960735689191982, + "grad_norm": 1.478288173675537, + "learning_rate": 1.940660884577894e-08, + "loss": 0.6508, + "step": 9640 + }, + { + "epoch": 0.9971068402562513, + "grad_norm": 1.7626447677612305, + "learning_rate": 1.053661447877885e-08, + "loss": 0.5287, + "step": 9650 + }, + { + "epoch": 0.9981401115933044, + "grad_norm": 1.4299039840698242, + "learning_rate": 4.354435154813042e-09, + "loss": 0.6035, + "step": 9660 + }, + { + "epoch": 0.9991733829303575, + "grad_norm": 0.8753514885902405, + "learning_rate": 8.601373429339976e-10, + "loss": 0.6028, + "step": 9670 + }, + { + "epoch": 1.0, + "eval_loss": 2.729660987854004, + "eval_runtime": 5409.1379, + "eval_samples_per_second": 9.159, + "eval_steps_per_second": 0.286, + "step": 9678 + } + ], + "logging_steps": 10, + "max_steps": 9678, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/Ins/checkpoint-9678/zero_to_fp32.py b/Ins/checkpoint-9678/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..e69ecd9acb5a235ffbf927091051106d902b3d39 --- /dev/null +++ b/Ins/checkpoint-9678/zero_to_fp32.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import json +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + state_dict_split = split_torch_state_dict_into_shards(state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors} + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard, output_path, metadata={"format": "pt"}) + else: + torch.save(shard, output_path) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/Ins/config.json b/Ins/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bf9f4aa8c6fb464585e94fe5fe52dc6b5db8eddf --- /dev/null +++ b/Ins/config.json @@ -0,0 +1,111 @@ +{ + "_name_or_path": "/home/sgugger/tmp/llama/llama-7b/", + "architectures": [ + "LlamaWithRQ" + ], + "args": { + "add_prefix": false, + "base_model": "/home/jovyan/workspace/Llama-7b", + "batch_size": 1024, + "bf16": true, + "bn": false, + "ckpt_dir": "", + "data_path": "/home/jovyan/workspace", + "dataloader_num_workers": 4, + "dataloader_prefetch_factor": 2, + "dataset": "Instruments", + "deepspeed": "./config/ds_z2_bf16.json", + "device": "cuda:1", + "dropout_prob": 0.0, + "e_dim": 32, + "epochs": 1, + "eval_step": 50, + "fp16": false, + "gradient_accumulation_steps": 2, + "his_sep": ", ", + "index_file": ".index.json", + "kmeans_init": false, + "kmeans_iters": 100, + "layers": [ + 2048, + 1024, + 512, + 256, + 128, + 64 + ], + "learner": "AdamW", + "learning_rate": 0.0005, + "logging_step": 10, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_modules_to_save": "embed_tokens,lm_head", + "lora_r": 8, + "lora_target_modules": "q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj", + "loss_type": "mse", + "lr": 0.001, + "lr_scheduler_type": "cosine", + "max_his_len": 20, + "model_max_length": 1024, + "num_emb_list": [ + 256, + 256, + 256, + 256 + ], + "num_workers": 4, + "only_train_response": true, + "optim": "adamw_torch", + "output_dir": "./Ins", + "per_device_batch_size": 8, + "quant_loss_weight": 1.0, + "remove_unused_columns": false, + "resume_from_checkpoint": null, + "rqvae_model": "/home/jovyan/workspace/LC-Rec/index/Ins/Apr-04-2025_07-12-04/best_collision_model.pth", + "sample_valid": true, + "save_and_eval_steps": 1000, + "save_and_eval_strategy": "epoch", + "seed": 42, + "sk_epsilons": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "sk_iters": 50, + "tasks": "seqrec,itemsearch,inters2title,inters2description,preferenceobtain,item2index,index2item,intertitles2item,query2item", + "train_data_sample_num": "0,0,0,0,0,0,0,0,0", + "train_prompt_sample_num": "1,1,1,1,1,1,1,1,1", + "valid_prompt_id": 0, + "valid_prompt_sample_num": 2, + "warmup": 5, + "warmup_ratio": 0.01, + "weight_decay": 0.01 + }, + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 2048, + "max_sequence_length": 2048, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 32, + "pad_token_id": 0, + "pretraining_tp": 1, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.45.2", + "use_cache": false, + "vocab_size": 33024 +} diff --git a/Ins/finetune/README.md b/Ins/finetune/README.md new file mode 100644 index 0000000000000000000000000000000000000000..26e9c9e58630d1aac7f47250ed6e0ca9a15a251f --- /dev/null +++ b/Ins/finetune/README.md @@ -0,0 +1,202 @@ +--- +base_model: /home/jovyan/workspace/Llama-7b +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.15.1 \ No newline at end of file diff --git a/Ins/finetune/adapter_config.json b/Ins/finetune/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6ecc29f5416e21ecf7303e956889199bbb0d5587 --- /dev/null +++ b/Ins/finetune/adapter_config.json @@ -0,0 +1,42 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/home/jovyan/workspace/Llama-7b", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [ + "embed_tokens", + "lm_head" + ], + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "v_proj", + "q_proj", + "k_proj", + "down_proj", + "gate_proj", + "o_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/Ins/finetune/added_tokens.json b/Ins/finetune/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..d98719c951a52f071b6df8945c2261f26413dff1 --- /dev/null +++ b/Ins/finetune/added_tokens.json @@ -0,0 +1,1026 @@ +{ + "": 32000, + "": 32100, + "": 32101, + "": 32102, + "": 32103, + "": 32104, + "": 32105, + "": 32106, + "": 32107, + "": 32108, + "": 32109, + "": 32010, + "": 32110, + "": 32111, + "": 32112, + "": 32113, + "": 32114, + "": 32115, + "": 32116, + "": 32117, + "": 32118, + "": 32119, + "": 32011, + "": 32120, + "": 32121, + "": 32122, + "": 32123, + "": 32124, + "": 32125, + "": 32126, + "": 32127, + "": 32128, + "": 32129, + "": 32012, + "": 32130, + "": 32131, + "": 32132, + "": 32133, + "": 32134, + "": 32135, + "": 32136, + "": 32137, + "": 32138, + "": 32139, + "": 32013, + "": 32140, + "": 32141, + "": 32142, + "": 32143, + "": 32144, + "": 32145, + "": 32146, + "": 32147, + "": 32148, + "": 32149, + "": 32014, + "": 32150, + "": 32151, + "": 32152, + "": 32153, + "": 32154, + "": 32155, + "": 32156, + "": 32157, + "": 32158, + "": 32159, + "": 32015, + "": 32160, + "": 32161, + "": 32162, + "": 32163, + "": 32164, + "": 32165, + "": 32166, + "": 32167, + "": 32168, + "": 32169, + "": 32016, + "": 32170, + "": 32171, + "": 32172, + "": 32173, + "": 32174, + "": 32175, + "": 32176, + "": 32177, + "": 32178, + "": 32179, + "": 32017, + "": 32180, + "": 32181, + "": 32182, + "": 32183, + "": 32184, + "": 32185, + "": 32186, + "": 32187, + "": 32188, + "": 32189, + "": 32018, + "": 32190, + "": 32191, + "": 32192, + "": 32193, + "": 32194, + "": 32195, + "": 32196, + "": 32197, + "": 32198, + "": 32199, + "": 32019, + "": 32001, + "": 32200, + "": 32201, + "": 32202, + "": 32203, + "": 32204, + "": 32205, + "": 32206, + "": 32207, + "": 32208, + "": 32209, + "": 32020, + "": 32210, + "": 32211, + "": 32212, + "": 32213, + "": 32214, + "": 32215, + "": 32216, + "": 32217, + "": 32218, + "": 32219, + "": 32021, + "": 32220, + "": 32221, + "": 32222, + "": 32223, + "": 32224, + "": 32225, + "": 32226, + "": 32227, + "": 32228, + "": 32229, + "": 32022, + "": 32230, + "": 32231, + "": 32232, + "": 32233, + "": 32234, + "": 32235, + "": 32236, + "": 32237, + "": 32238, + "": 32239, + "": 32023, + "": 32240, + "": 32241, + "": 32242, + "": 32243, + "": 32244, + "": 32245, + "": 32246, + "": 32247, + "": 32248, + "": 32249, + "": 32024, + "": 32250, + "": 32251, + "": 32252, + "": 32253, + "": 32254, + "": 32255, + "": 32025, + "": 32026, + "": 32027, + "": 32028, + "": 32029, + "": 32002, + "": 32030, + "": 32031, + "": 32032, + "": 32033, + "": 32034, + "": 32035, + "": 32036, + "": 32037, + "": 32038, + "": 32039, + "": 32003, + "": 32040, + "": 32041, + "": 32042, + "": 32043, + "": 32044, + "": 32045, + "": 32046, + "": 32047, + "": 32048, + "": 32049, + "": 32004, + "": 32050, + "": 32051, + "": 32052, + "": 32053, + "": 32054, + "": 32055, + "": 32056, + "": 32057, + "": 32058, + "": 32059, + "": 32005, + "": 32060, + "": 32061, + "": 32062, + "": 32063, + "": 32064, + "": 32065, + "": 32066, + "": 32067, + "": 32068, + "": 32069, + "": 32006, + "": 32070, + "": 32071, + "": 32072, + "": 32073, + "": 32074, + "": 32075, + "": 32076, + "": 32077, + "": 32078, + "": 32079, + "": 32007, + "": 32080, + "": 32081, + "": 32082, + "": 32083, + "": 32084, + "": 32085, + "": 32086, + "": 32087, + "": 32088, + "": 32089, + "": 32008, + "": 32090, + "": 32091, + "": 32092, + "": 32093, + "": 32094, + "": 32095, + "": 32096, + "": 32097, + "": 32098, + "": 32099, + "": 32009, + "": 32256, + "": 32356, + "": 32357, + "": 32358, + "": 32359, + "": 32360, + "": 32361, + "": 32362, + "": 32363, + "": 32364, + "": 32365, + "": 32266, + "": 32366, + "": 32367, + "": 32368, + "": 32369, + "": 32370, + "": 32371, + "": 32372, + "": 32373, + "": 32374, + "": 32375, + "": 32267, + "": 32376, + "": 32377, + "": 32378, + "": 32379, + "": 32380, + "": 32381, + "": 32382, + "": 32383, + "": 32384, + "": 32385, + "": 32268, + "": 32386, + "": 32387, + "": 32388, + "": 32389, + "": 32390, + "": 32391, + "": 32392, + "": 32393, + "": 32394, + "": 32395, + "": 32269, + "": 32396, + "": 32397, + "": 32398, + "": 32399, + "": 32400, + "": 32401, + "": 32402, + "": 32403, + "": 32404, + "": 32405, + "": 32270, + "": 32406, + "": 32407, + "": 32408, + "": 32409, + "": 32410, + "": 32411, + "": 32412, + "": 32413, + "": 32414, + "": 32415, + "": 32271, + "": 32416, + "": 32417, + "": 32418, + "": 32419, + "": 32420, + "": 32421, + "": 32422, + "": 32423, + "": 32424, + "": 32425, + "": 32272, + "": 32426, + "": 32427, + "": 32428, + "": 32429, + "": 32430, + "": 32431, + "": 32432, + "": 32433, + "": 32434, + "": 32435, + "": 32273, + "": 32436, + "": 32437, + "": 32438, + "": 32439, + "": 32440, + "": 32441, + "": 32442, + "": 32443, + "": 32444, + "": 32445, + "": 32274, + "": 32446, + "": 32447, + "": 32448, + "": 32449, + "": 32450, + "": 32451, + "": 32452, + "": 32453, + "": 32454, + "": 32455, + "": 32275, + "": 32257, + "": 32456, + "": 32457, + "": 32458, + "": 32459, + "": 32460, + "": 32461, + "": 32462, + "": 32463, + "": 32464, + "": 32465, + "": 32276, + "": 32466, + "": 32467, + "": 32468, + "": 32469, + "": 32470, + "": 32471, + "": 32472, + "": 32473, + "": 32474, + "": 32475, + "": 32277, + "": 32476, + "": 32477, + "": 32478, + "": 32479, + "": 32480, + "": 32481, + "": 32482, + "": 32483, + "": 32484, + "": 32485, + "": 32278, + "": 32486, + "": 32487, + "": 32488, + "": 32489, + "": 32490, + "": 32491, + "": 32492, + "": 32493, + "": 32494, + "": 32495, + "": 32279, + "": 32496, + "": 32497, + "": 32498, + "": 32499, + "": 32500, + "": 32501, + "": 32502, + "": 32503, + "": 32504, + "": 32505, + "": 32280, + "": 32506, + "": 32507, + "": 32508, + "": 32509, + "": 32510, + "": 32511, + "": 32281, + "": 32282, + "": 32283, + "": 32284, + "": 32285, + "": 32258, + "": 32286, + "": 32287, + "": 32288, + "": 32289, + "": 32290, + "": 32291, + "": 32292, + "": 32293, + "": 32294, + "": 32295, + "": 32259, + "": 32296, + "": 32297, + "": 32298, + "": 32299, + "": 32300, + "": 32301, + "": 32302, + "": 32303, + "": 32304, + "": 32305, + "": 32260, + "": 32306, + "": 32307, + "": 32308, + "": 32309, + "": 32310, + "": 32311, + "": 32312, + "": 32313, + "": 32314, + "": 32315, + "": 32261, + "": 32316, + "": 32317, + "": 32318, + "": 32319, + "": 32320, + "": 32321, + "": 32322, + "": 32323, + "": 32324, + "": 32325, + "": 32262, + "": 32326, + "": 32327, + "": 32328, + "": 32329, + "": 32330, + "": 32331, + "": 32332, + "": 32333, + "": 32334, + "": 32335, + "": 32263, + "": 32336, + "": 32337, + "": 32338, + "": 32339, + "": 32340, + "": 32341, + "": 32342, + "": 32343, + "": 32344, + "": 32345, + "": 32264, + "": 32346, + "": 32347, + "": 32348, + "": 32349, + "": 32350, + "": 32351, + "": 32352, + "": 32353, + "": 32354, + "": 32355, + "": 32265, + "": 32512, + "": 32612, + "": 32613, + "": 32614, + "": 32615, + "": 32616, + "": 32617, + "": 32618, + "": 32619, + "": 32620, + "": 32621, + "": 32522, + "": 32622, + "": 32623, + "": 32624, + "": 32625, + "": 32626, + "": 32627, + "": 32628, + "": 32629, + "": 32630, + "": 32631, + "": 32523, + "": 32632, + "": 32633, + "": 32634, + "": 32635, + "": 32636, + "": 32637, + "": 32638, + "": 32639, + "": 32640, + "": 32641, + "": 32524, + "": 32642, + "": 32643, + "": 32644, + "": 32645, + "": 32646, + "": 32647, + "": 32648, + "": 32649, + "": 32650, + "": 32651, + "": 32525, + "": 32652, + "": 32653, + "": 32654, + "": 32655, + "": 32656, + "": 32657, + "": 32658, + "": 32659, + "": 32660, + "": 32661, + "": 32526, + "": 32662, + "": 32663, + "": 32664, + "": 32665, + "": 32666, + "": 32667, + "": 32668, + "": 32669, + "": 32670, + "": 32671, + "": 32527, + "": 32672, + "": 32673, + "": 32674, + "": 32675, + "": 32676, + "": 32677, + "": 32678, + "": 32679, + "": 32680, + "": 32681, + "": 32528, + "": 32682, + "": 32683, + "": 32684, + "": 32685, + "": 32686, + "": 32687, + "": 32688, + "": 32689, + "": 32690, + "": 32691, + "": 32529, + "": 32692, + "": 32693, + "": 32694, + "": 32695, + "": 32696, + "": 32697, + "": 32698, + "": 32699, + "": 32700, + "": 32701, + "": 32530, + "": 32702, + "": 32703, + "": 32704, + "": 32705, + "": 32706, + "": 32707, + "": 32708, + "": 32709, + "": 32710, + "": 32711, + "": 32531, + "": 32513, + "": 32712, + "": 32713, + "": 32714, + "": 32715, + "": 32716, + "": 32717, + "": 32718, + "": 32719, + "": 32720, + "": 32721, + "": 32532, + "": 32722, + "": 32723, + "": 32724, + "": 32725, + "": 32726, + "": 32727, + "": 32728, + "": 32729, + "": 32730, + "": 32731, + "": 32533, + "": 32732, + "": 32733, + "": 32734, + "": 32735, + "": 32736, + "": 32737, + "": 32738, + "": 32739, + "": 32740, + "": 32741, + "": 32534, + "": 32742, + "": 32743, + "": 32744, + "": 32745, + "": 32746, + "": 32747, + "": 32748, + "": 32749, + "": 32750, + "": 32751, + "": 32535, + "": 32752, + "": 32753, + "": 32754, + "": 32755, + "": 32756, + "": 32757, + "": 32758, + "": 32759, + "": 32760, + "": 32761, + "": 32536, + "": 32762, + "": 32763, + "": 32764, + "": 32765, + "": 32766, + "": 32767, + "": 32537, + "": 32538, + "": 32539, + "": 32540, + "": 32541, + "": 32514, + "": 32542, + "": 32543, + "": 32544, + "": 32545, + "": 32546, + "": 32547, + "": 32548, + "": 32549, + "": 32550, + "": 32551, + "": 32515, + "": 32552, + "": 32553, + "": 32554, + "": 32555, + "": 32556, + "": 32557, + "": 32558, + "": 32559, + "": 32560, + "": 32561, + "": 32516, + "": 32562, + "": 32563, + "": 32564, + "": 32565, + "": 32566, + "": 32567, + "": 32568, + "": 32569, + "": 32570, + "": 32571, + "": 32517, + "": 32572, + "": 32573, + "": 32574, + "": 32575, + "": 32576, + "": 32577, + "": 32578, + "": 32579, + "": 32580, + "": 32581, + "": 32518, + "": 32582, + "": 32583, + "": 32584, + "": 32585, + "": 32586, + "": 32587, + "": 32588, + "": 32589, + "": 32590, + "": 32591, + "": 32519, + "": 32592, + "": 32593, + "": 32594, + "": 32595, + "": 32596, + "": 32597, + "": 32598, + "": 32599, + "": 32600, + "": 32601, + "": 32520, + "": 32602, + "": 32603, + "": 32604, + "": 32605, + "": 32606, + "": 32607, + "": 32608, + "": 32609, + "": 32610, + "": 32611, + "": 32521, + "": 32768, + "": 32868, + "": 32869, + "": 32870, + "": 32871, + "": 32872, + "": 32873, + "": 32874, + "": 32875, + "": 32876, + "": 32877, + "": 32778, + "": 32878, + "": 32879, + "": 32880, + "": 32881, + "": 32882, + "": 32883, + "": 32884, + "": 32885, + "": 32886, + "": 32887, + "": 32779, + "": 32888, + "": 32889, + "": 32890, + "": 32891, + "": 32892, + "": 32893, + "": 32894, + "": 32895, + "": 32896, + "": 32897, + "": 32780, + "": 32898, + "": 32899, + "": 32900, + "": 32901, + "": 32902, + "": 32903, + "": 32904, + "": 32905, + "": 32906, + "": 32907, + "": 32781, + "": 32908, + "": 32909, + "": 32910, + "": 32911, + "": 32912, + "": 32913, + "": 32914, + "": 32915, + "": 32916, + "": 32917, + "": 32782, + "": 32918, + "": 32919, + "": 32920, + "": 32921, + "": 32922, + "": 32923, + "": 32924, + "": 32925, + "": 32926, + "": 32927, + "": 32783, + "": 32928, + "": 32929, + "": 32930, + "": 32931, + "": 32932, + "": 32933, + "": 32934, + "": 32935, + "": 32936, + "": 32937, + "": 32784, + "": 32938, + "": 32939, + "": 32940, + "": 32941, + "": 32942, + "": 32943, + "": 32944, + "": 32945, + "": 32946, + "": 32947, + "": 32785, + "": 32948, + "": 32949, + "": 32950, + "": 32951, + "": 32952, + "": 32953, + "": 32954, + "": 32955, + "": 32956, + "": 32957, + "": 32786, + "": 32958, + "": 32959, + "": 32960, + "": 32961, + "": 32962, + "": 32963, + "": 32964, + "": 32965, + "": 32966, + "": 32967, + "": 32787, + "": 32769, + "": 32968, + "": 32969, + "": 32970, + "": 32971, + "": 32972, + "": 32973, + "": 32974, + "": 32975, + "": 32976, + "": 32977, + "": 32788, + "": 32978, + "": 32979, + "": 32980, + "": 32981, + "": 32982, + "": 32983, + "": 32984, + "": 32985, + "": 32986, + "": 32987, + "": 32789, + "": 32988, + "": 32989, + "": 32990, + "": 32991, + "": 32992, + "": 32993, + "": 32994, + "": 32995, + "": 32996, + "": 32997, + "": 32790, + "": 32998, + "": 32999, + "": 33000, + "": 33001, + "": 33002, + "": 33003, + "": 33004, + "": 33005, + "": 33006, + "": 33007, + "": 32791, + "": 33008, + "": 33009, + "": 33010, + "": 33011, + "": 33012, + "": 33013, + "": 33014, + "": 33015, + "": 33016, + "": 33017, + "": 32792, + "": 33018, + "": 33019, + "": 33020, + "": 33021, + "": 33022, + "": 33023, + "": 32793, + "": 32794, + "": 32795, + "": 32796, + "": 32797, + "": 32770, + "": 32798, + "": 32799, + "": 32800, + "": 32801, + "": 32802, + "": 32803, + "": 32804, + "": 32805, + "": 32806, + "": 32807, + "": 32771, + "": 32808, + "": 32809, + "": 32810, + "": 32811, + "": 32812, + "": 32813, + "": 32814, + "": 32815, + "": 32816, + "": 32817, + "": 32772, + "": 32818, + "": 32819, + "": 32820, + "": 32821, + "": 32822, + "": 32823, + "": 32824, + "": 32825, + "": 32826, + "": 32827, + "": 32773, + "": 32828, + "": 32829, + "": 32830, + "": 32831, + "": 32832, + "": 32833, + "": 32834, + "": 32835, + "": 32836, + "": 32837, + "": 32774, + "": 32838, + "": 32839, + "": 32840, + "": 32841, + "": 32842, + "": 32843, + "": 32844, + "": 32845, + "": 32846, + "": 32847, + "": 32775, + "": 32848, + "": 32849, + "": 32850, + "": 32851, + "": 32852, + "": 32853, + "": 32854, + "": 32855, + "": 32856, + "": 32857, + "": 32776, + "": 32858, + "": 32859, + "": 32860, + "": 32861, + "": 32862, + "": 32863, + "": 32864, + "": 32865, + "": 32866, + "": 32867, + "": 32777 +} diff --git a/Ins/finetune/eval_result.json b/Ins/finetune/eval_result.json new file mode 100644 index 0000000000000000000000000000000000000000..e46e0dc9f324a061ebd49b8d5de7d459a31c3abd --- /dev/null +++ b/Ins/finetune/eval_result.json @@ -0,0 +1,47 @@ +{ + "test_prompt_ids": "all", + "mean_results": { + "hit@1": 0.06059260455352818, + "hit@5": 0.08478658700683567, + "hit@10": 0.10541471553904946, + "ndcg@5": 0.07265244992899023, + "ndcg@10": 0.07924828323692125 + }, + "min_results": { + "hit@1": 0.06055223639593089, + "hit@5": 0.08436944937833037, + "hit@10": 0.10528015501372517, + "ndcg@5": 0.07247135835974591, + "ndcg@10": 0.07913786546067342 + }, + "max_results": { + "hit@1": 0.06067334086872275, + "hit@5": 0.0852171806878734, + "hit@10": 0.10560310027450347, + "ndcg@5": 0.07292103606730017, + "ndcg@10": 0.079352714069041 + }, + "all_prompt_results": [ + { + "hit@1": 0.06055223639593089, + "hit@5": 0.08477313095430325, + "hit@10": 0.10528015501372517, + "ndcg@5": 0.07256495535992459, + "ndcg@10": 0.07913786546067342 + }, + { + "hit@1": 0.06055223639593089, + "hit@5": 0.08436944937833037, + "hit@10": 0.10560310027450347, + "ndcg@5": 0.07247135835974591, + "ndcg@10": 0.07925427018104932 + }, + { + "hit@1": 0.06067334086872275, + "hit@5": 0.0852171806878734, + "hit@10": 0.10536089132891975, + "ndcg@5": 0.07292103606730017, + "ndcg@10": 0.079352714069041 + } + ] +} \ No newline at end of file diff --git a/Ins/finetune/log.txt b/Ins/finetune/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a1b5899491d998b6ece61ba03599ebe2227fbfd2 --- /dev/null +++ b/Ins/finetune/log.txt @@ -0,0 +1,645 @@ +W0405 07:40:59.464000 140086321264448 torch/distributed/run.py:779] +W0405 07:40:59.464000 140086321264448 torch/distributed/run.py:779] ***************************************** +W0405 07:40:59.464000 140086321264448 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0405 07:40:59.464000 140086321264448 torch/distributed/run.py:779] ***************************************** +{'seed': 42, 'base_model': '../llama-7b/', 'output_dir': '/home/jovyan/workspace/ed2/Ins/finetune', 'data_path': '/home/jovyan/workspace', 'tasks': 'seqrec,itemsearch,preferenceobtain,item2index,index2item,fusionseqrec', 'train_data_sample_num': '0,0,0,0,0,0', 'dataset': 'Instruments', 'index_file': '/home/jovyan/workspace/ed2/Ins/indices.json', 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': 2, 'max_his_len': 20, 'add_prefix': False, 'his_sep': ', ', 'only_train_response': True, 'train_prompt_sample_num': '1,1,1,1,1,1', 'valid_prompt_id': 0, 'sample_valid': True, 'valid_prompt_sample_num': 2, 'optim': 'adamw_torch', 'epochs': 1, 'learning_rate': 0.0005, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 2, 'logging_step': 10, 'model_max_length': 2048, 'weight_decay': 0.01, 'lora_r': 8, 'lora_alpha': 32, 'lora_dropout': 0.05, 'lora_target_modules': 'q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj', 'lora_modules_to_save': 'embed_tokens,lm_head', 'resume_from_checkpoint': None, 'warmup_ratio': 0.01, 'lr_scheduler_type': 'cosine', 'save_and_eval_strategy': 'epoch', 'save_and_eval_steps': 1000, 'fp16': False, 'bf16': True, 'deepspeed': './config/ds_z2_bf16.json', 'remove_unused_columns': False, 're_index': 0, 'ckpt_path': '/home/jovyan/workspace/ed2/Ins'} +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message + Loading checkpoint shards: 0%| | 0/2 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message + Loading checkpoint shards: 0%| | 0/2 [00:00 + server.login(sender, mail_code) + File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 739, in login + (code, resp) = self.auth( + ^^^^^^^^^^ + File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 642, in auth + (code, resp) = self.docmd("AUTH", mechanism + " " + response) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 432, in docmd + return self.getreply() + ^^^^^^^^^^^^^^^ + File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 405, in getreply + raise SMTPServerDisconnected("Connection unexpectedly closed") +smtplib.SMTPServerDisconnected: Connection unexpectedly closed +[rank0]: Traceback (most recent call last): +[rank0]: File "/home/jovyan/workspace/ed2/fine-tune.py", line 152, in +[rank0]: server.login(sender, mail_code) +[rank0]: File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 739, in login +[rank0]: (code, resp) = self.auth( +[rank0]: ^^^^^^^^^^ +[rank0]: File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 642, in auth +[rank0]: (code, resp) = self.docmd("AUTH", mechanism + " " + response) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 432, in docmd +[rank0]: return self.getreply() +[rank0]: ^^^^^^^^^^^^^^^ +[rank0]: File "/opt/saturncloud/envs/saturn/lib/python3.12/smtplib.py", line 405, in getreply +[rank0]: raise SMTPServerDisconnected("Connection unexpectedly closed") +[rank0]: smtplib.SMTPServerDisconnected: Connection unexpectedly closed +E0405 11:06:18.093000 140086321264448 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 0 (pid: 595118) of binary: /opt/saturncloud/envs/saturn/bin/python +Traceback (most recent call last): + File "/opt/saturncloud/envs/saturn/bin/torchrun", line 33, in + sys.exit(load_entry_point('torch==2.4.0', 'console_scripts', 'torchrun')()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/saturncloud/envs/saturn/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/opt/saturncloud/envs/saturn/lib/python3.12/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/opt/saturncloud/envs/saturn/lib/python3.12/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/opt/saturncloud/envs/saturn/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/saturncloud/envs/saturn/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +fine-tune.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-04-05_11:06:18 + host : w-zjin-mmrs-23fd93126a624b1689d7f490c793415e-96b4c7955-l8vpn + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 595118) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/Ins/finetune/special_tokens_map.json b/Ins/finetune/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..f928b2409a393d47ce0d9fe519f17e048a471eca --- /dev/null +++ b/Ins/finetune/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/Ins/finetune/tokenizer_config.json b/Ins/finetune/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..af2930551fdc0a50e9fb9b05d370d7743e0ade97 --- /dev/null +++ b/Ins/finetune/tokenizer_config.json @@ -0,0 +1,8236 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": true, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + }, + "32000": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32001": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32002": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32003": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32004": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32005": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32006": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32007": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32008": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32009": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32010": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32011": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32012": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32013": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32014": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32015": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32016": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32017": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32018": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32019": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32020": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32021": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32022": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32023": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32024": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32025": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32026": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32027": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32028": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32029": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32030": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32031": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32032": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32033": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32034": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32035": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32036": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32037": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32038": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32039": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32040": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32041": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32042": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32043": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32044": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32045": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32046": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32047": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32048": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32049": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32050": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32051": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32052": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32053": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32054": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32055": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32056": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32057": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32058": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32059": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32060": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32061": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32062": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32063": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32064": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32065": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32066": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32067": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32068": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32069": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32070": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32071": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32072": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32073": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32074": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32075": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32076": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32077": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32078": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32079": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32080": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32081": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32082": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32083": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32084": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32085": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32086": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32087": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32088": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32089": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32090": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32091": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32092": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32093": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32094": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32095": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32096": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32097": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32098": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32099": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32100": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32101": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32102": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32103": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32104": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32105": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32106": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32107": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32108": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32109": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32110": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32111": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32112": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32113": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32114": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32115": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32116": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32117": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32118": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32119": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32120": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32121": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32122": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32123": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32124": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32125": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32126": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32127": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32128": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32129": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32130": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32131": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32132": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32133": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32134": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32135": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32136": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32137": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32138": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32139": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32140": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32141": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32142": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32143": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32144": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32145": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32146": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32147": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32148": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32149": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32150": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32151": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32152": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32153": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32154": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32155": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32156": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32157": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32158": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32159": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32160": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32161": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32162": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32163": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32164": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32165": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32166": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32167": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32168": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32169": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32170": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32171": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32172": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32173": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32174": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32175": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32176": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32177": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32178": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32179": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32180": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32181": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32182": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32183": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32184": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32185": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32186": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32187": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32188": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32189": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32190": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32191": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32192": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32193": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32194": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32195": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32196": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32197": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32198": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32199": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32200": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32201": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32202": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32203": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32204": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32205": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32206": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32207": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32208": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32209": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32210": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32211": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32212": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32213": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32214": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32215": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32216": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32217": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32218": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32219": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32220": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32221": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32222": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32223": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32224": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32225": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32226": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32227": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32228": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32229": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32230": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32231": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32232": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32233": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32234": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32235": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32236": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32237": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32238": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32239": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32240": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32241": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32242": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32243": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32244": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32245": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32246": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32247": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32248": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32249": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32250": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32251": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32252": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32253": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32254": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32255": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32256": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32257": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32258": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32259": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32260": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32261": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32262": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32263": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32264": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32265": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32266": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32267": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32268": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32269": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32270": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32271": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32272": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32273": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32274": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32275": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32276": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32277": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32278": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32279": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32280": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32281": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32282": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32283": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32284": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32285": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32286": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32287": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32288": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32289": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32290": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32291": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32292": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32293": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32294": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32295": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32296": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32297": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32298": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32299": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32300": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32301": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32302": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32303": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32304": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32305": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32306": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32307": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32308": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32309": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32310": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32311": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32312": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32313": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32314": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32315": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32316": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32317": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32318": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32319": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32320": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32321": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32322": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32323": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32324": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32325": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32326": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32327": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32328": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32329": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32330": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32331": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32332": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32333": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32334": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32335": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32336": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32337": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32338": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32339": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32340": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32341": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32342": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32343": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32344": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32345": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32346": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32347": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32348": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32349": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32350": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32351": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32352": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32353": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32354": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32355": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32356": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32357": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32358": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32359": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32360": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32361": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32362": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32363": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32364": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32365": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32366": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32367": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32368": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32369": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32370": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32371": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32372": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32373": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32374": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32375": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32376": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32377": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32378": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32379": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32380": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32381": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32382": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32383": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32384": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32385": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32386": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32387": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32388": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32389": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32390": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32391": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32392": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32393": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32394": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32395": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32396": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32397": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32398": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32399": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32400": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32401": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32402": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32403": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32404": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32405": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32406": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32407": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32408": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32409": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32410": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32411": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32412": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32413": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32414": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32415": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32416": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32417": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32418": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32419": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32420": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32421": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32422": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32423": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32424": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32425": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32426": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32427": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32428": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32429": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32430": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32431": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32432": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32433": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32434": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32435": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32436": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32437": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32438": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32439": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32440": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32441": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32442": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32443": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32444": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32445": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32446": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32447": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32448": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32449": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32450": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32451": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32452": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32453": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32454": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32455": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32456": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32457": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32458": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32459": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32460": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32461": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32462": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32463": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32464": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32465": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32466": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32467": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32468": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32469": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32470": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32471": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32472": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32473": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32474": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32475": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32476": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32477": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32478": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32479": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32480": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32481": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32482": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32483": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32484": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32485": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32486": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32487": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32488": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32489": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32490": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32491": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32492": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32493": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32494": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32495": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32496": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32497": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32498": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32499": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32500": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32501": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32502": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32503": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32504": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32505": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32506": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32507": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32508": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32509": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32510": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32511": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32512": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32513": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32514": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32515": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32516": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32517": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32518": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32519": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32520": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32521": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32522": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32523": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32524": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32525": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32526": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32527": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32528": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32529": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32530": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32531": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32532": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32533": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32534": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32535": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32536": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32537": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32538": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32539": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32540": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32541": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32542": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32543": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32544": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32545": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32546": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32547": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32548": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32549": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32550": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32551": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32552": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32553": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32554": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32555": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32556": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32557": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32558": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32559": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32560": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32561": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32562": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32563": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32564": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32565": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32566": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32567": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32568": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32569": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32570": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32571": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32572": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32573": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32574": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32575": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32576": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32577": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32578": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32579": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32580": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32581": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32582": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32583": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32584": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32585": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32586": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32587": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32588": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32589": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32590": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32591": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32592": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32593": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32594": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32595": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32596": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32597": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32598": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32599": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32600": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32601": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32602": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32603": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32604": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32605": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32606": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32607": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32608": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32609": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32610": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32611": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32612": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32613": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32614": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32615": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32616": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32617": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32618": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32619": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32620": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32621": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32622": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32623": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32624": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32625": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32626": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32627": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32628": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32629": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32630": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32631": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32632": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32633": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32634": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32635": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32636": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32637": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32638": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32639": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32640": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32641": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32642": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32643": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32644": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32645": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32646": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32647": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32648": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32649": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32650": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32651": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32652": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32653": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32654": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32655": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32656": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32657": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32658": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32659": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32660": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32661": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32662": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32663": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32664": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32665": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32666": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32667": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32668": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32669": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32670": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32671": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32672": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32673": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32674": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32675": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32676": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32677": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32678": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32679": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32680": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32681": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32682": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32683": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32684": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32685": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32686": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32687": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32688": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32689": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32690": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32691": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32692": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32693": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32694": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32695": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32696": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32697": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32698": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32699": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32700": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32701": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32702": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32703": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32704": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32705": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32706": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32707": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32708": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32709": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32710": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32711": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32712": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32713": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32714": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32715": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32716": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32717": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32718": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32719": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32720": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32721": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32722": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32723": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32724": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32725": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32726": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32727": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32728": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32729": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32730": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32731": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32732": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32733": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32734": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32735": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32736": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32737": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32738": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32739": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32740": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32741": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32742": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32743": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32744": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32745": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32746": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32747": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32748": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32749": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32750": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32751": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32752": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32753": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32754": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32755": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32756": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32757": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32758": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32759": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32760": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32761": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32762": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32763": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32764": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32765": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32766": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32767": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32768": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32769": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32770": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32771": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32772": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32773": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32774": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32775": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32776": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32777": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32778": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32779": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32780": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32781": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32782": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32783": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32784": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32785": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32786": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32787": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32788": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32789": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32790": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32791": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32792": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32793": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32794": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32795": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32796": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32797": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32798": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32799": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32800": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32801": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32802": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32803": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32804": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32805": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32806": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32807": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32808": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32809": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32810": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32811": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32812": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32813": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32814": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32815": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32816": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32817": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32818": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32819": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32820": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32821": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32822": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32823": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32824": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32825": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32826": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32827": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32828": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32829": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32830": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32831": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32832": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32833": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32834": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32835": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32836": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32837": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32838": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32839": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32840": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32841": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32842": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32843": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32844": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32845": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32846": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32847": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32848": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32849": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32850": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32851": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32852": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32853": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32854": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32855": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32856": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32857": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32858": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32859": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32860": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32861": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32862": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32863": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32864": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32865": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32866": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32867": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32868": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32869": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32870": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32871": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32872": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32873": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32874": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32875": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32876": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32877": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32878": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32879": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32880": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32881": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32882": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32883": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32884": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32885": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32886": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32887": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32888": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32889": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32890": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32891": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32892": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32893": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32894": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32895": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32896": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32897": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32898": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32899": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32900": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32901": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32902": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32903": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32904": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32905": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32906": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32907": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32908": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32909": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32910": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32911": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32912": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32913": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32914": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32915": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32916": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32917": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32918": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32919": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32920": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32921": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32922": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32923": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32924": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32925": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32926": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32927": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32928": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32929": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32930": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32931": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32932": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32933": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32934": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32935": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32936": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32937": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32938": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32939": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32940": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32941": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32942": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32943": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32944": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32945": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32946": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32947": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32948": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32949": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32950": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32951": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32952": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32953": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32954": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32955": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32956": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32957": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32958": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32959": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32960": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32961": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32962": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32963": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32964": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32965": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32966": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32967": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32968": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32969": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32970": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32971": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32972": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32973": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32974": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32975": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32976": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32977": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32978": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32979": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32980": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32981": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32982": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32983": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32984": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32985": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32986": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32987": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32988": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32989": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32990": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32991": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32992": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32993": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32994": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32995": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32996": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32997": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32998": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32999": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33000": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33001": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33002": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33003": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33004": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33005": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33006": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33007": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33008": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33009": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33010": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33011": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33012": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33013": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33014": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33015": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33016": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33017": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33018": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33019": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33020": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33021": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33022": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33023": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\\n\\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don\\'t know the answer to a question, please don\\'t share false information.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\n' + content.strip() + '\\n<>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/Ins/finetune/trainer_state.json b/Ins/finetune/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5a9b2b37b72cd9a1fbbfc4863068ef3d994f71ae --- /dev/null +++ b/Ins/finetune/trainer_state.json @@ -0,0 +1,3682 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9999039292919589, + "eval_steps": 1000, + "global_step": 5204, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0019214141608223652, + "grad_norm": 2.4305918216705322, + "learning_rate": 9.433962264150944e-05, + "loss": 0.837, + "step": 10 + }, + { + "epoch": 0.0038428283216447303, + "grad_norm": 1.3495147228240967, + "learning_rate": 0.00018867924528301889, + "loss": 0.7681, + "step": 20 + }, + { + "epoch": 0.005764242482467096, + "grad_norm": 0.7792263627052307, + "learning_rate": 0.0002830188679245283, + "loss": 0.7964, + "step": 30 + }, + { + "epoch": 0.007685656643289461, + "grad_norm": 2.0215744972229004, + "learning_rate": 0.00037735849056603777, + "loss": 0.8555, + "step": 40 + }, + { + "epoch": 0.009607070804111826, + "grad_norm": 0.9997712969779968, + "learning_rate": 0.0004716981132075472, + "loss": 0.973, + "step": 50 + }, + { + "epoch": 0.011528484964934192, + "grad_norm": 1.6805115938186646, + "learning_rate": 0.0004999977216414076, + "loss": 0.8087, + "step": 60 + }, + { + "epoch": 0.013449899125756557, + "grad_norm": 1.888655424118042, + "learning_rate": 0.0004999865624339867, + "loss": 0.8988, + "step": 70 + }, + { + "epoch": 0.015371313286578921, + "grad_norm": 1.7796673774719238, + "learning_rate": 0.0004999661043182869, + "loss": 0.9576, + "step": 80 + }, + { + "epoch": 0.017292727447401287, + "grad_norm": 1.4261183738708496, + "learning_rate": 0.000499936348055302, + "loss": 0.9808, + "step": 90 + }, + { + "epoch": 0.019214141608223653, + "grad_norm": 1.119388461112976, + "learning_rate": 0.0004998972947518943, + "loss": 0.9937, + "step": 100 + }, + { + "epoch": 0.021135555769046018, + "grad_norm": 1.7162309885025024, + "learning_rate": 0.0004998489458607546, + "loss": 0.8548, + "step": 110 + }, + { + "epoch": 0.023056969929868384, + "grad_norm": 2.1326487064361572, + "learning_rate": 0.0004997913031803468, + "loss": 1.0711, + "step": 120 + }, + { + "epoch": 0.02497838409069075, + "grad_norm": 2.0284371376037598, + "learning_rate": 0.0004997243688548423, + "loss": 1.1614, + "step": 130 + }, + { + "epoch": 0.026899798251513115, + "grad_norm": 1.6004656553268433, + "learning_rate": 0.0004996481453740396, + "loss": 0.9902, + "step": 140 + }, + { + "epoch": 0.02882121241233548, + "grad_norm": 2.1148738861083984, + "learning_rate": 0.0004995626355732716, + "loss": 0.9928, + "step": 150 + }, + { + "epoch": 0.030742626573157843, + "grad_norm": 1.4066400527954102, + "learning_rate": 0.0004994678426333004, + "loss": 1.0203, + "step": 160 + }, + { + "epoch": 0.03266404073398021, + "grad_norm": 2.228640079498291, + "learning_rate": 0.0004993637700801992, + "loss": 0.8989, + "step": 170 + }, + { + "epoch": 0.034585454894802574, + "grad_norm": 1.9775664806365967, + "learning_rate": 0.0004992504217852204, + "loss": 0.9383, + "step": 180 + }, + { + "epoch": 0.03650686905562494, + "grad_norm": 1.725915551185608, + "learning_rate": 0.0004991278019646523, + "loss": 0.8669, + "step": 190 + }, + { + "epoch": 0.038428283216447305, + "grad_norm": 1.2805640697479248, + "learning_rate": 0.0004989959151796617, + "loss": 0.9812, + "step": 200 + }, + { + "epoch": 0.04034969737726967, + "grad_norm": 1.4963750839233398, + "learning_rate": 0.0004988547663361251, + "loss": 1.0042, + "step": 210 + }, + { + "epoch": 0.042271111538092036, + "grad_norm": 2.081295967102051, + "learning_rate": 0.000498704360684445, + "loss": 0.9171, + "step": 220 + }, + { + "epoch": 0.0441925256989144, + "grad_norm": 1.5910736322402954, + "learning_rate": 0.0004985447038193558, + "loss": 1.0254, + "step": 230 + }, + { + "epoch": 0.04611393985973677, + "grad_norm": 1.3932862281799316, + "learning_rate": 0.0004983758016797147, + "loss": 0.9257, + "step": 240 + }, + { + "epoch": 0.04803535402055913, + "grad_norm": 1.7471644878387451, + "learning_rate": 0.0004981976605482817, + "loss": 0.95, + "step": 250 + }, + { + "epoch": 0.0499567681813815, + "grad_norm": 1.6283310651779175, + "learning_rate": 0.0004980102870514847, + "loss": 1.0308, + "step": 260 + }, + { + "epoch": 0.051878182342203864, + "grad_norm": 3.03869366645813, + "learning_rate": 0.0004978136881591746, + "loss": 1.118, + "step": 270 + }, + { + "epoch": 0.05379959650302623, + "grad_norm": 1.8459982872009277, + "learning_rate": 0.0004976078711843645, + "loss": 1.0017, + "step": 280 + }, + { + "epoch": 0.055721010663848596, + "grad_norm": 2.3289153575897217, + "learning_rate": 0.0004973928437829586, + "loss": 1.0451, + "step": 290 + }, + { + "epoch": 0.05764242482467096, + "grad_norm": 2.016827344894409, + "learning_rate": 0.0004971686139534673, + "loss": 1.0557, + "step": 300 + }, + { + "epoch": 0.05956383898549332, + "grad_norm": 2.2310760021209717, + "learning_rate": 0.0004969351900367092, + "loss": 1.0835, + "step": 310 + }, + { + "epoch": 0.061485253146315685, + "grad_norm": 2.030564308166504, + "learning_rate": 0.0004966925807155016, + "loss": 0.984, + "step": 320 + }, + { + "epoch": 0.06340666730713805, + "grad_norm": 2.2611382007598877, + "learning_rate": 0.0004964407950143367, + "loss": 0.9892, + "step": 330 + }, + { + "epoch": 0.06532808146796042, + "grad_norm": 1.6385694742202759, + "learning_rate": 0.0004961798422990465, + "loss": 1.0289, + "step": 340 + }, + { + "epoch": 0.06724949562878278, + "grad_norm": 2.0164542198181152, + "learning_rate": 0.0004959097322764543, + "loss": 1.0196, + "step": 350 + }, + { + "epoch": 0.06917090978960515, + "grad_norm": 2.2265095710754395, + "learning_rate": 0.0004956304749940134, + "loss": 1.009, + "step": 360 + }, + { + "epoch": 0.07109232395042751, + "grad_norm": 2.171071767807007, + "learning_rate": 0.0004953420808394334, + "loss": 1.0981, + "step": 370 + }, + { + "epoch": 0.07301373811124988, + "grad_norm": 2.4532387256622314, + "learning_rate": 0.0004950445605402943, + "loss": 1.1436, + "step": 380 + }, + { + "epoch": 0.07493515227207224, + "grad_norm": 1.9529705047607422, + "learning_rate": 0.0004947379251636468, + "loss": 1.0647, + "step": 390 + }, + { + "epoch": 0.07685656643289461, + "grad_norm": 2.0442216396331787, + "learning_rate": 0.000494422186115601, + "loss": 0.9467, + "step": 400 + }, + { + "epoch": 0.07877798059371698, + "grad_norm": 1.7892729043960571, + "learning_rate": 0.0004940973551409018, + "loss": 1.0112, + "step": 410 + }, + { + "epoch": 0.08069939475453934, + "grad_norm": 1.5844688415527344, + "learning_rate": 0.0004937634443224925, + "loss": 0.9157, + "step": 420 + }, + { + "epoch": 0.08262080891536171, + "grad_norm": 1.737804651260376, + "learning_rate": 0.0004934204660810651, + "loss": 0.9012, + "step": 430 + }, + { + "epoch": 0.08454222307618407, + "grad_norm": 1.868697166442871, + "learning_rate": 0.000493068433174598, + "loss": 0.9935, + "step": 440 + }, + { + "epoch": 0.08646363723700644, + "grad_norm": 2.2820777893066406, + "learning_rate": 0.000492707358697882, + "loss": 1.1136, + "step": 450 + }, + { + "epoch": 0.0883850513978288, + "grad_norm": 1.7056925296783447, + "learning_rate": 0.000492337256082033, + "loss": 1.096, + "step": 460 + }, + { + "epoch": 0.09030646555865117, + "grad_norm": 1.9547524452209473, + "learning_rate": 0.0004919581390939917, + "loss": 0.9342, + "step": 470 + }, + { + "epoch": 0.09222787971947354, + "grad_norm": 1.2121398448944092, + "learning_rate": 0.0004915700218360126, + "loss": 1.062, + "step": 480 + }, + { + "epoch": 0.0941492938802959, + "grad_norm": 2.3924214839935303, + "learning_rate": 0.0004911729187451389, + "loss": 0.9367, + "step": 490 + }, + { + "epoch": 0.09607070804111827, + "grad_norm": 1.8371268510818481, + "learning_rate": 0.0004907668445926656, + "loss": 0.9993, + "step": 500 + }, + { + "epoch": 0.09799212220194063, + "grad_norm": 1.3057355880737305, + "learning_rate": 0.0004903518144835895, + "loss": 1.0558, + "step": 510 + }, + { + "epoch": 0.099913536362763, + "grad_norm": 1.72185480594635, + "learning_rate": 0.0004899278438560481, + "loss": 1.0392, + "step": 520 + }, + { + "epoch": 0.10183495052358536, + "grad_norm": 2.33323335647583, + "learning_rate": 0.0004894949484807453, + "loss": 1.0486, + "step": 530 + }, + { + "epoch": 0.10375636468440773, + "grad_norm": 2.24397349357605, + "learning_rate": 0.0004890531444603637, + "loss": 1.1116, + "step": 540 + }, + { + "epoch": 0.1056777788452301, + "grad_norm": 1.5562959909439087, + "learning_rate": 0.0004886024482289674, + "loss": 0.9655, + "step": 550 + }, + { + "epoch": 0.10759919300605246, + "grad_norm": 1.6606842279434204, + "learning_rate": 0.00048814287655138877, + "loss": 0.9895, + "step": 560 + }, + { + "epoch": 0.10952060716687483, + "grad_norm": 1.591304898262024, + "learning_rate": 0.00048767444652260644, + "loss": 1.0343, + "step": 570 + }, + { + "epoch": 0.11144202132769719, + "grad_norm": 1.592618465423584, + "learning_rate": 0.0004871971755671084, + "loss": 1.0607, + "step": 580 + }, + { + "epoch": 0.11336343548851956, + "grad_norm": 1.6015795469284058, + "learning_rate": 0.00048671108143824414, + "loss": 1.1387, + "step": 590 + }, + { + "epoch": 0.11528484964934192, + "grad_norm": 2.1288461685180664, + "learning_rate": 0.0004862161822175646, + "loss": 1.0174, + "step": 600 + }, + { + "epoch": 0.11720626381016427, + "grad_norm": 2.2891428470611572, + "learning_rate": 0.0004857124963141489, + "loss": 1.0713, + "step": 610 + }, + { + "epoch": 0.11912767797098664, + "grad_norm": 2.3416545391082764, + "learning_rate": 0.0004852000424639205, + "loss": 0.9538, + "step": 620 + }, + { + "epoch": 0.121049092131809, + "grad_norm": 1.7260591983795166, + "learning_rate": 0.00048467883972894897, + "loss": 1.0152, + "step": 630 + }, + { + "epoch": 0.12297050629263137, + "grad_norm": 2.208760976791382, + "learning_rate": 0.00048414890749674247, + "loss": 0.9158, + "step": 640 + }, + { + "epoch": 0.12489192045345374, + "grad_norm": 1.8179646730422974, + "learning_rate": 0.0004836102654795252, + "loss": 0.9743, + "step": 650 + }, + { + "epoch": 0.1268133346142761, + "grad_norm": 1.9145281314849854, + "learning_rate": 0.0004830629337135051, + "loss": 1.0373, + "step": 660 + }, + { + "epoch": 0.12873474877509847, + "grad_norm": 1.7451472282409668, + "learning_rate": 0.00048250693255812795, + "loss": 1.03, + "step": 670 + }, + { + "epoch": 0.13065616293592083, + "grad_norm": 1.9409160614013672, + "learning_rate": 0.0004819422826953204, + "loss": 1.0401, + "step": 680 + }, + { + "epoch": 0.1325775770967432, + "grad_norm": 1.6609559059143066, + "learning_rate": 0.00048136900512872063, + "loss": 1.0893, + "step": 690 + }, + { + "epoch": 0.13449899125756556, + "grad_norm": 1.9034476280212402, + "learning_rate": 0.0004807871211828969, + "loss": 0.9952, + "step": 700 + }, + { + "epoch": 0.13642040541838793, + "grad_norm": 2.3018155097961426, + "learning_rate": 0.00048019665250255417, + "loss": 1.1016, + "step": 710 + }, + { + "epoch": 0.1383418195792103, + "grad_norm": 1.745120644569397, + "learning_rate": 0.00047959762105172964, + "loss": 0.9148, + "step": 720 + }, + { + "epoch": 0.14026323374003266, + "grad_norm": 1.6242783069610596, + "learning_rate": 0.00047899004911297496, + "loss": 0.9516, + "step": 730 + }, + { + "epoch": 0.14218464790085503, + "grad_norm": 1.867432713508606, + "learning_rate": 0.00047837395928652785, + "loss": 1.1345, + "step": 740 + }, + { + "epoch": 0.1441060620616774, + "grad_norm": 2.4577126502990723, + "learning_rate": 0.00047774937448947125, + "loss": 1.1109, + "step": 750 + }, + { + "epoch": 0.14602747622249976, + "grad_norm": 2.3080718517303467, + "learning_rate": 0.00047711631795488093, + "loss": 1.1036, + "step": 760 + }, + { + "epoch": 0.14794889038332212, + "grad_norm": 1.4562822580337524, + "learning_rate": 0.0004764748132309612, + "loss": 1.1009, + "step": 770 + }, + { + "epoch": 0.1498703045441445, + "grad_norm": 2.0178232192993164, + "learning_rate": 0.000475824884180169, + "loss": 1.054, + "step": 780 + }, + { + "epoch": 0.15179171870496685, + "grad_norm": 1.6681246757507324, + "learning_rate": 0.0004751665549783264, + "loss": 0.9606, + "step": 790 + }, + { + "epoch": 0.15371313286578922, + "grad_norm": 1.3220267295837402, + "learning_rate": 0.0004744998501137209, + "loss": 0.9446, + "step": 800 + }, + { + "epoch": 0.15563454702661159, + "grad_norm": 2.040184259414673, + "learning_rate": 0.0004738247943861953, + "loss": 1.1417, + "step": 810 + }, + { + "epoch": 0.15755596118743395, + "grad_norm": 2.3848249912261963, + "learning_rate": 0.00047314141290622435, + "loss": 1.032, + "step": 820 + }, + { + "epoch": 0.15947737534825632, + "grad_norm": 1.8507578372955322, + "learning_rate": 0.00047244973109398115, + "loss": 1.0843, + "step": 830 + }, + { + "epoch": 0.16139878950907868, + "grad_norm": 1.6274956464767456, + "learning_rate": 0.0004717497746783916, + "loss": 1.089, + "step": 840 + }, + { + "epoch": 0.16332020366990105, + "grad_norm": 1.817091703414917, + "learning_rate": 0.0004710415696961773, + "loss": 1.0112, + "step": 850 + }, + { + "epoch": 0.16524161783072341, + "grad_norm": 2.092291831970215, + "learning_rate": 0.0004703251424908868, + "loss": 1.0262, + "step": 860 + }, + { + "epoch": 0.16716303199154578, + "grad_norm": 1.6193962097167969, + "learning_rate": 0.0004696005197119159, + "loss": 1.0422, + "step": 870 + }, + { + "epoch": 0.16908444615236815, + "grad_norm": 1.929657220840454, + "learning_rate": 0.00046886772831351663, + "loss": 0.9062, + "step": 880 + }, + { + "epoch": 0.1710058603131905, + "grad_norm": 2.1643612384796143, + "learning_rate": 0.0004681267955537941, + "loss": 1.0763, + "step": 890 + }, + { + "epoch": 0.17292727447401288, + "grad_norm": 1.4907313585281372, + "learning_rate": 0.0004673777489936927, + "loss": 0.9692, + "step": 900 + }, + { + "epoch": 0.17484868863483524, + "grad_norm": 1.7910531759262085, + "learning_rate": 0.0004666206164959712, + "loss": 0.9425, + "step": 910 + }, + { + "epoch": 0.1767701027956576, + "grad_norm": 2.4189772605895996, + "learning_rate": 0.00046585542622416587, + "loss": 0.9799, + "step": 920 + }, + { + "epoch": 0.17869151695647997, + "grad_norm": 1.827263355255127, + "learning_rate": 0.0004650822066415434, + "loss": 0.9834, + "step": 930 + }, + { + "epoch": 0.18061293111730234, + "grad_norm": 0.9529337882995605, + "learning_rate": 0.0004643009865100415, + "loss": 0.9683, + "step": 940 + }, + { + "epoch": 0.1825343452781247, + "grad_norm": 1.812353491783142, + "learning_rate": 0.0004635117948891997, + "loss": 1.063, + "step": 950 + }, + { + "epoch": 0.18445575943894707, + "grad_norm": 1.5051437616348267, + "learning_rate": 0.00046271466113507776, + "loss": 0.9554, + "step": 960 + }, + { + "epoch": 0.18637717359976944, + "grad_norm": 2.0191311836242676, + "learning_rate": 0.0004619096148991641, + "loss": 1.1198, + "step": 970 + }, + { + "epoch": 0.1882985877605918, + "grad_norm": 1.7233866453170776, + "learning_rate": 0.0004610966861272728, + "loss": 1.0903, + "step": 980 + }, + { + "epoch": 0.19022000192141417, + "grad_norm": 1.3007336854934692, + "learning_rate": 0.0004602759050584295, + "loss": 1.0139, + "step": 990 + }, + { + "epoch": 0.19214141608223653, + "grad_norm": 2.3556885719299316, + "learning_rate": 0.00045944730222374693, + "loss": 1.0511, + "step": 1000 + }, + { + "epoch": 0.1940628302430589, + "grad_norm": 2.345998525619507, + "learning_rate": 0.00045861090844528864, + "loss": 0.9954, + "step": 1010 + }, + { + "epoch": 0.19598424440388126, + "grad_norm": 1.4576616287231445, + "learning_rate": 0.00045776675483492305, + "loss": 1.0297, + "step": 1020 + }, + { + "epoch": 0.19790565856470363, + "grad_norm": 2.5236847400665283, + "learning_rate": 0.000456914872793166, + "loss": 0.9905, + "step": 1030 + }, + { + "epoch": 0.199827072725526, + "grad_norm": 1.475441575050354, + "learning_rate": 0.0004560552940080126, + "loss": 1.1028, + "step": 1040 + }, + { + "epoch": 0.20174848688634836, + "grad_norm": 2.0014760494232178, + "learning_rate": 0.00045518805045375855, + "loss": 0.892, + "step": 1050 + }, + { + "epoch": 0.20366990104717073, + "grad_norm": 1.9783899784088135, + "learning_rate": 0.0004543131743898109, + "loss": 1.0747, + "step": 1060 + }, + { + "epoch": 0.2055913152079931, + "grad_norm": 2.0599184036254883, + "learning_rate": 0.0004534306983594879, + "loss": 1.1359, + "step": 1070 + }, + { + "epoch": 0.20751272936881546, + "grad_norm": 1.8748066425323486, + "learning_rate": 0.0004525406551888087, + "loss": 1.0495, + "step": 1080 + }, + { + "epoch": 0.20943414352963782, + "grad_norm": 1.6007455587387085, + "learning_rate": 0.0004516430779852721, + "loss": 1.1541, + "step": 1090 + }, + { + "epoch": 0.2113555576904602, + "grad_norm": 1.7732211351394653, + "learning_rate": 0.00045073800013662493, + "loss": 0.94, + "step": 1100 + }, + { + "epoch": 0.21327697185128255, + "grad_norm": 1.3590902090072632, + "learning_rate": 0.00044982545530962046, + "loss": 1.0359, + "step": 1110 + }, + { + "epoch": 0.21519838601210492, + "grad_norm": 1.4337408542633057, + "learning_rate": 0.00044890547744876573, + "loss": 1.0596, + "step": 1120 + }, + { + "epoch": 0.21711980017292729, + "grad_norm": 2.3610267639160156, + "learning_rate": 0.0004479781007750593, + "loss": 1.0121, + "step": 1130 + }, + { + "epoch": 0.21904121433374965, + "grad_norm": 1.9422202110290527, + "learning_rate": 0.0004470433597847176, + "loss": 1.0459, + "step": 1140 + }, + { + "epoch": 0.22096262849457202, + "grad_norm": 1.4544517993927002, + "learning_rate": 0.0004461012892478927, + "loss": 0.9524, + "step": 1150 + }, + { + "epoch": 0.22288404265539438, + "grad_norm": 1.988907814025879, + "learning_rate": 0.000445151924207378, + "loss": 0.9969, + "step": 1160 + }, + { + "epoch": 0.22480545681621675, + "grad_norm": 2.1969475746154785, + "learning_rate": 0.0004441952999773056, + "loss": 1.0165, + "step": 1170 + }, + { + "epoch": 0.2267268709770391, + "grad_norm": 1.824702501296997, + "learning_rate": 0.000443231452141832, + "loss": 0.9324, + "step": 1180 + }, + { + "epoch": 0.22864828513786148, + "grad_norm": 1.3211947679519653, + "learning_rate": 0.00044226041655381465, + "loss": 0.9587, + "step": 1190 + }, + { + "epoch": 0.23056969929868384, + "grad_norm": 1.7261728048324585, + "learning_rate": 0.00044128222933347857, + "loss": 1.0037, + "step": 1200 + }, + { + "epoch": 0.2324911134595062, + "grad_norm": 1.6653329133987427, + "learning_rate": 0.0004402969268670725, + "loss": 1.0027, + "step": 1210 + }, + { + "epoch": 0.23441252762032855, + "grad_norm": 1.6761009693145752, + "learning_rate": 0.00043930454580551543, + "loss": 0.9722, + "step": 1220 + }, + { + "epoch": 0.2363339417811509, + "grad_norm": 2.3884928226470947, + "learning_rate": 0.0004383051230630335, + "loss": 1.0587, + "step": 1230 + }, + { + "epoch": 0.23825535594197328, + "grad_norm": 1.994282841682434, + "learning_rate": 0.0004372986958157864, + "loss": 0.9573, + "step": 1240 + }, + { + "epoch": 0.24017677010279564, + "grad_norm": 1.8947217464447021, + "learning_rate": 0.0004362853015004853, + "loss": 0.9498, + "step": 1250 + }, + { + "epoch": 0.242098184263618, + "grad_norm": 1.383719801902771, + "learning_rate": 0.00043526497781299923, + "loss": 0.985, + "step": 1260 + }, + { + "epoch": 0.24401959842444038, + "grad_norm": 1.3979140520095825, + "learning_rate": 0.00043423776270695393, + "loss": 0.991, + "step": 1270 + }, + { + "epoch": 0.24594101258526274, + "grad_norm": 2.1677207946777344, + "learning_rate": 0.0004332036943923192, + "loss": 1.0134, + "step": 1280 + }, + { + "epoch": 0.2478624267460851, + "grad_norm": 2.216585636138916, + "learning_rate": 0.0004321628113339885, + "loss": 1.0522, + "step": 1290 + }, + { + "epoch": 0.24978384090690747, + "grad_norm": 1.8316318988800049, + "learning_rate": 0.000431115152250347, + "loss": 0.9927, + "step": 1300 + }, + { + "epoch": 0.25170525506772984, + "grad_norm": 1.802751898765564, + "learning_rate": 0.0004300607561118325, + "loss": 0.9922, + "step": 1310 + }, + { + "epoch": 0.2536266692285522, + "grad_norm": 1.3108793497085571, + "learning_rate": 0.00042899966213948496, + "loss": 1.0054, + "step": 1320 + }, + { + "epoch": 0.25554808338937457, + "grad_norm": 1.4082355499267578, + "learning_rate": 0.00042793190980348797, + "loss": 0.906, + "step": 1330 + }, + { + "epoch": 0.25746949755019694, + "grad_norm": 1.4897574186325073, + "learning_rate": 0.00042685753882170063, + "loss": 1.0401, + "step": 1340 + }, + { + "epoch": 0.2593909117110193, + "grad_norm": 1.6914223432540894, + "learning_rate": 0.00042577658915817987, + "loss": 1.0751, + "step": 1350 + }, + { + "epoch": 0.26131232587184167, + "grad_norm": 1.5143028497695923, + "learning_rate": 0.0004246891010216939, + "loss": 0.9363, + "step": 1360 + }, + { + "epoch": 0.26323374003266403, + "grad_norm": 1.7084685564041138, + "learning_rate": 0.0004235951148642269, + "loss": 0.8922, + "step": 1370 + }, + { + "epoch": 0.2651551541934864, + "grad_norm": 2.340919256210327, + "learning_rate": 0.00042249467137947386, + "loss": 1.0042, + "step": 1380 + }, + { + "epoch": 0.26707656835430876, + "grad_norm": 1.8660237789154053, + "learning_rate": 0.00042138781150132703, + "loss": 0.9955, + "step": 1390 + }, + { + "epoch": 0.26899798251513113, + "grad_norm": 1.997502326965332, + "learning_rate": 0.0004202745764023536, + "loss": 0.9646, + "step": 1400 + }, + { + "epoch": 0.2709193966759535, + "grad_norm": 1.2619147300720215, + "learning_rate": 0.0004191550074922634, + "loss": 0.9677, + "step": 1410 + }, + { + "epoch": 0.27284081083677586, + "grad_norm": 1.921871542930603, + "learning_rate": 0.0004180291464163696, + "loss": 0.8853, + "step": 1420 + }, + { + "epoch": 0.2747622249975982, + "grad_norm": 1.8054934740066528, + "learning_rate": 0.0004168970350540384, + "loss": 0.9755, + "step": 1430 + }, + { + "epoch": 0.2766836391584206, + "grad_norm": 1.6009109020233154, + "learning_rate": 0.00041575871551713254, + "loss": 0.8524, + "step": 1440 + }, + { + "epoch": 0.27860505331924296, + "grad_norm": 1.9609386920928955, + "learning_rate": 0.00041461423014844354, + "loss": 1.0005, + "step": 1450 + }, + { + "epoch": 0.2805264674800653, + "grad_norm": 1.6398521661758423, + "learning_rate": 0.00041346362152011763, + "loss": 1.0373, + "step": 1460 + }, + { + "epoch": 0.2824478816408877, + "grad_norm": 1.6447991132736206, + "learning_rate": 0.00041230693243207185, + "loss": 1.0963, + "step": 1470 + }, + { + "epoch": 0.28436929580171005, + "grad_norm": 1.813947081565857, + "learning_rate": 0.0004111442059104017, + "loss": 0.9324, + "step": 1480 + }, + { + "epoch": 0.2862907099625324, + "grad_norm": 1.4735851287841797, + "learning_rate": 0.00040997548520578097, + "loss": 0.9437, + "step": 1490 + }, + { + "epoch": 0.2882121241233548, + "grad_norm": 1.5962815284729004, + "learning_rate": 0.000408800813791853, + "loss": 0.9427, + "step": 1500 + }, + { + "epoch": 0.29013353828417715, + "grad_norm": 1.527431845664978, + "learning_rate": 0.00040762023536361334, + "loss": 0.9776, + "step": 1510 + }, + { + "epoch": 0.2920549524449995, + "grad_norm": 1.4146003723144531, + "learning_rate": 0.00040643379383578453, + "loss": 0.9488, + "step": 1520 + }, + { + "epoch": 0.2939763666058219, + "grad_norm": 1.4645968675613403, + "learning_rate": 0.0004052415333411824, + "loss": 0.8507, + "step": 1530 + }, + { + "epoch": 0.29589778076664425, + "grad_norm": 1.2212328910827637, + "learning_rate": 0.00040404349822907484, + "loss": 0.99, + "step": 1540 + }, + { + "epoch": 0.2978191949274666, + "grad_norm": 2.1287434101104736, + "learning_rate": 0.0004028397330635315, + "loss": 0.9587, + "step": 1550 + }, + { + "epoch": 0.299740609088289, + "grad_norm": 1.9199310541152954, + "learning_rate": 0.0004016302826217667, + "loss": 0.9378, + "step": 1560 + }, + { + "epoch": 0.30166202324911134, + "grad_norm": 1.958397626876831, + "learning_rate": 0.0004004151918924734, + "loss": 1.0496, + "step": 1570 + }, + { + "epoch": 0.3035834374099337, + "grad_norm": 1.6783572435379028, + "learning_rate": 0.0003991945060741502, + "loss": 0.9261, + "step": 1580 + }, + { + "epoch": 0.3055048515707561, + "grad_norm": 2.240941047668457, + "learning_rate": 0.0003979682705734194, + "loss": 0.966, + "step": 1590 + }, + { + "epoch": 0.30742626573157844, + "grad_norm": 1.0998966693878174, + "learning_rate": 0.0003967365310033385, + "loss": 0.9686, + "step": 1600 + }, + { + "epoch": 0.3093476798924008, + "grad_norm": 1.5177958011627197, + "learning_rate": 0.00039549933318170353, + "loss": 0.9887, + "step": 1610 + }, + { + "epoch": 0.31126909405322317, + "grad_norm": 1.780063509941101, + "learning_rate": 0.0003942567231293442, + "loss": 0.8932, + "step": 1620 + }, + { + "epoch": 0.31319050821404554, + "grad_norm": 1.9309372901916504, + "learning_rate": 0.0003930087470684127, + "loss": 0.9206, + "step": 1630 + }, + { + "epoch": 0.3151119223748679, + "grad_norm": 1.5048413276672363, + "learning_rate": 0.00039175545142066385, + "loss": 0.9546, + "step": 1640 + }, + { + "epoch": 0.31703333653569027, + "grad_norm": 1.0540335178375244, + "learning_rate": 0.00039049688280572847, + "loss": 0.8159, + "step": 1650 + }, + { + "epoch": 0.31895475069651263, + "grad_norm": 2.461320638656616, + "learning_rate": 0.00038923308803937956, + "loss": 0.9886, + "step": 1660 + }, + { + "epoch": 0.320876164857335, + "grad_norm": 1.532655954360962, + "learning_rate": 0.0003879641141317903, + "loss": 0.9431, + "step": 1670 + }, + { + "epoch": 0.32279757901815737, + "grad_norm": 1.5244719982147217, + "learning_rate": 0.0003866900082857857, + "loss": 0.943, + "step": 1680 + }, + { + "epoch": 0.32471899317897973, + "grad_norm": 1.2937613725662231, + "learning_rate": 0.00038541081789508693, + "loss": 0.9269, + "step": 1690 + }, + { + "epoch": 0.3266404073398021, + "grad_norm": 1.5547471046447754, + "learning_rate": 0.00038412659054254796, + "loss": 0.8873, + "step": 1700 + }, + { + "epoch": 0.32856182150062446, + "grad_norm": 1.1971843242645264, + "learning_rate": 0.00038283737399838583, + "loss": 0.9211, + "step": 1710 + }, + { + "epoch": 0.33048323566144683, + "grad_norm": 1.9691673517227173, + "learning_rate": 0.0003815432162184037, + "loss": 0.8766, + "step": 1720 + }, + { + "epoch": 0.3324046498222692, + "grad_norm": 1.8342334032058716, + "learning_rate": 0.0003802441653422073, + "loss": 0.9308, + "step": 1730 + }, + { + "epoch": 0.33432606398309156, + "grad_norm": 1.1400641202926636, + "learning_rate": 0.0003789402696914136, + "loss": 0.9256, + "step": 1740 + }, + { + "epoch": 0.3362474781439139, + "grad_norm": 1.6318907737731934, + "learning_rate": 0.0003776315777678537, + "loss": 0.8789, + "step": 1750 + }, + { + "epoch": 0.3381688923047363, + "grad_norm": 1.1623560190200806, + "learning_rate": 0.00037631813825176913, + "loss": 1.022, + "step": 1760 + }, + { + "epoch": 0.34009030646555866, + "grad_norm": 1.0850412845611572, + "learning_rate": 0.000375, + "loss": 0.8747, + "step": 1770 + }, + { + "epoch": 0.342011720626381, + "grad_norm": 1.9052928686141968, + "learning_rate": 0.00037367721204416873, + "loss": 0.8603, + "step": 1780 + }, + { + "epoch": 0.3439331347872034, + "grad_norm": 1.469117283821106, + "learning_rate": 0.0003723498235888556, + "loss": 0.9025, + "step": 1790 + }, + { + "epoch": 0.34585454894802575, + "grad_norm": 2.0990898609161377, + "learning_rate": 0.0003710178840097685, + "loss": 0.8507, + "step": 1800 + }, + { + "epoch": 0.3477759631088481, + "grad_norm": 1.6102617979049683, + "learning_rate": 0.0003696814428519064, + "loss": 0.9469, + "step": 1810 + }, + { + "epoch": 0.3496973772696705, + "grad_norm": 1.8937904834747314, + "learning_rate": 0.0003683405498277164, + "loss": 0.9119, + "step": 1820 + }, + { + "epoch": 0.35161879143049285, + "grad_norm": 1.446227788925171, + "learning_rate": 0.0003669952548152443, + "loss": 0.8413, + "step": 1830 + }, + { + "epoch": 0.3535402055913152, + "grad_norm": 1.6752564907073975, + "learning_rate": 0.00036564560785627974, + "loss": 0.9523, + "step": 1840 + }, + { + "epoch": 0.3554616197521376, + "grad_norm": 1.836383581161499, + "learning_rate": 0.00036429165915449416, + "loss": 0.889, + "step": 1850 + }, + { + "epoch": 0.35738303391295995, + "grad_norm": 1.7696865797042847, + "learning_rate": 0.0003629334590735738, + "loss": 1.0661, + "step": 1860 + }, + { + "epoch": 0.3593044480737823, + "grad_norm": 1.602179765701294, + "learning_rate": 0.0003615710581353463, + "loss": 0.8948, + "step": 1870 + }, + { + "epoch": 0.3612258622346047, + "grad_norm": 1.8728015422821045, + "learning_rate": 0.0003602045070179009, + "loss": 0.9631, + "step": 1880 + }, + { + "epoch": 0.36314727639542704, + "grad_norm": 2.071622848510742, + "learning_rate": 0.0003588338565537039, + "loss": 0.941, + "step": 1890 + }, + { + "epoch": 0.3650686905562494, + "grad_norm": 1.5780361890792847, + "learning_rate": 0.0003574591577277076, + "loss": 0.8537, + "step": 1900 + }, + { + "epoch": 0.3669901047170718, + "grad_norm": 1.4528398513793945, + "learning_rate": 0.0003560804616754536, + "loss": 0.8613, + "step": 1910 + }, + { + "epoch": 0.36891151887789414, + "grad_norm": 1.888594627380371, + "learning_rate": 0.0003546978196811711, + "loss": 0.9791, + "step": 1920 + }, + { + "epoch": 0.3708329330387165, + "grad_norm": 1.3231236934661865, + "learning_rate": 0.00035331128317586885, + "loss": 0.9041, + "step": 1930 + }, + { + "epoch": 0.37275434719953887, + "grad_norm": 1.3922632932662964, + "learning_rate": 0.0003519209037354222, + "loss": 0.9247, + "step": 1940 + }, + { + "epoch": 0.37467576136036124, + "grad_norm": 1.7650400400161743, + "learning_rate": 0.0003505267330786547, + "loss": 0.9474, + "step": 1950 + }, + { + "epoch": 0.3765971755211836, + "grad_norm": 1.8358489274978638, + "learning_rate": 0.0003491288230654138, + "loss": 0.8982, + "step": 1960 + }, + { + "epoch": 0.37851858968200597, + "grad_norm": 1.263236165046692, + "learning_rate": 0.00034772722569464276, + "loss": 0.8903, + "step": 1970 + }, + { + "epoch": 0.38044000384282833, + "grad_norm": 1.6004908084869385, + "learning_rate": 0.00034632199310244535, + "loss": 0.9012, + "step": 1980 + }, + { + "epoch": 0.3823614180036507, + "grad_norm": 1.7554082870483398, + "learning_rate": 0.00034491317756014706, + "loss": 0.9149, + "step": 1990 + }, + { + "epoch": 0.38428283216447306, + "grad_norm": 1.0712970495224, + "learning_rate": 0.00034350083147235077, + "loss": 1.0325, + "step": 2000 + }, + { + "epoch": 0.38620424632529543, + "grad_norm": 1.5798851251602173, + "learning_rate": 0.0003420850073749872, + "loss": 1.0006, + "step": 2010 + }, + { + "epoch": 0.3881256604861178, + "grad_norm": 1.782596230506897, + "learning_rate": 0.00034066575793336075, + "loss": 0.8972, + "step": 2020 + }, + { + "epoch": 0.39004707464694016, + "grad_norm": 1.332011342048645, + "learning_rate": 0.0003392431359401906, + "loss": 0.8255, + "step": 2030 + }, + { + "epoch": 0.3919684888077625, + "grad_norm": 1.5429998636245728, + "learning_rate": 0.0003378171943136469, + "loss": 0.8661, + "step": 2040 + }, + { + "epoch": 0.3938899029685849, + "grad_norm": 1.1828477382659912, + "learning_rate": 0.0003363879860953822, + "loss": 0.9084, + "step": 2050 + }, + { + "epoch": 0.39581131712940726, + "grad_norm": 1.411515712738037, + "learning_rate": 0.0003349555644485585, + "loss": 0.8647, + "step": 2060 + }, + { + "epoch": 0.3977327312902296, + "grad_norm": 1.8091164827346802, + "learning_rate": 0.00033351998265586987, + "loss": 0.9742, + "step": 2070 + }, + { + "epoch": 0.399654145451052, + "grad_norm": 1.3836525678634644, + "learning_rate": 0.00033208129411756024, + "loss": 0.8556, + "step": 2080 + }, + { + "epoch": 0.40157555961187436, + "grad_norm": 1.6256176233291626, + "learning_rate": 0.00033063955234943705, + "loss": 0.8681, + "step": 2090 + }, + { + "epoch": 0.4034969737726967, + "grad_norm": 1.7528786659240723, + "learning_rate": 0.0003291948109808809, + "loss": 0.9599, + "step": 2100 + }, + { + "epoch": 0.4054183879335191, + "grad_norm": 1.83562433719635, + "learning_rate": 0.0003277471237528502, + "loss": 0.8709, + "step": 2110 + }, + { + "epoch": 0.40733980209434145, + "grad_norm": 1.7219703197479248, + "learning_rate": 0.0003262965445158823, + "loss": 0.9668, + "step": 2120 + }, + { + "epoch": 0.4092612162551638, + "grad_norm": 1.4686826467514038, + "learning_rate": 0.0003248431272280908, + "loss": 0.9449, + "step": 2130 + }, + { + "epoch": 0.4111826304159862, + "grad_norm": 1.539268970489502, + "learning_rate": 0.0003233869259531577, + "loss": 0.925, + "step": 2140 + }, + { + "epoch": 0.41310404457680855, + "grad_norm": 1.6110196113586426, + "learning_rate": 0.0003219279948583229, + "loss": 0.8146, + "step": 2150 + }, + { + "epoch": 0.4150254587376309, + "grad_norm": 1.3205841779708862, + "learning_rate": 0.00032046638821236914, + "loss": 0.9522, + "step": 2160 + }, + { + "epoch": 0.4169468728984533, + "grad_norm": 2.021388530731201, + "learning_rate": 0.00031900216038360313, + "loss": 0.8603, + "step": 2170 + }, + { + "epoch": 0.41886828705927565, + "grad_norm": 1.2504761219024658, + "learning_rate": 0.00031753536583783374, + "loss": 0.9003, + "step": 2180 + }, + { + "epoch": 0.420789701220098, + "grad_norm": 2.206533432006836, + "learning_rate": 0.00031606605913634534, + "loss": 0.8686, + "step": 2190 + }, + { + "epoch": 0.4227111153809204, + "grad_norm": 1.5957015752792358, + "learning_rate": 0.00031459429493386863, + "loss": 0.8447, + "step": 2200 + }, + { + "epoch": 0.42463252954174274, + "grad_norm": 1.2046681642532349, + "learning_rate": 0.00031312012797654756, + "loss": 0.826, + "step": 2210 + }, + { + "epoch": 0.4265539437025651, + "grad_norm": 2.017876625061035, + "learning_rate": 0.00031164361309990283, + "loss": 0.8967, + "step": 2220 + }, + { + "epoch": 0.4284753578633875, + "grad_norm": 1.3104060888290405, + "learning_rate": 0.00031016480522679223, + "loss": 0.8396, + "step": 2230 + }, + { + "epoch": 0.43039677202420984, + "grad_norm": 1.9322564601898193, + "learning_rate": 0.00030868375936536754, + "loss": 0.9624, + "step": 2240 + }, + { + "epoch": 0.4323181861850322, + "grad_norm": 1.526458501815796, + "learning_rate": 0.00030720053060702835, + "loss": 0.9021, + "step": 2250 + }, + { + "epoch": 0.43423960034585457, + "grad_norm": 1.5850962400436401, + "learning_rate": 0.0003057151741243731, + "loss": 0.874, + "step": 2260 + }, + { + "epoch": 0.43616101450667694, + "grad_norm": 1.2977805137634277, + "learning_rate": 0.0003042277451691462, + "loss": 0.9072, + "step": 2270 + }, + { + "epoch": 0.4380824286674993, + "grad_norm": 2.240307092666626, + "learning_rate": 0.0003027382990701833, + "loss": 0.8864, + "step": 2280 + }, + { + "epoch": 0.44000384282832167, + "grad_norm": 1.734005093574524, + "learning_rate": 0.00030124689123135306, + "loss": 0.8596, + "step": 2290 + }, + { + "epoch": 0.44192525698914403, + "grad_norm": 1.654840350151062, + "learning_rate": 0.00029975357712949625, + "loss": 0.9219, + "step": 2300 + }, + { + "epoch": 0.4438466711499664, + "grad_norm": 1.530916690826416, + "learning_rate": 0.0002982584123123619, + "loss": 0.976, + "step": 2310 + }, + { + "epoch": 0.44576808531078876, + "grad_norm": 1.589181661605835, + "learning_rate": 0.00029676145239654144, + "loss": 0.8203, + "step": 2320 + }, + { + "epoch": 0.44768949947161113, + "grad_norm": 1.8620957136154175, + "learning_rate": 0.0002952627530653997, + "loss": 0.9294, + "step": 2330 + }, + { + "epoch": 0.4496109136324335, + "grad_norm": 1.6283154487609863, + "learning_rate": 0.00029376237006700366, + "loss": 1.0087, + "step": 2340 + }, + { + "epoch": 0.45153232779325586, + "grad_norm": 1.7942941188812256, + "learning_rate": 0.00029226035921204864, + "loss": 0.9341, + "step": 2350 + }, + { + "epoch": 0.4534537419540782, + "grad_norm": 1.7389261722564697, + "learning_rate": 0.00029075677637178243, + "loss": 0.8171, + "step": 2360 + }, + { + "epoch": 0.4553751561149006, + "grad_norm": 1.1926878690719604, + "learning_rate": 0.000289251677475927, + "loss": 0.8444, + "step": 2370 + }, + { + "epoch": 0.45729657027572296, + "grad_norm": 1.444225788116455, + "learning_rate": 0.0002877451185105979, + "loss": 0.7467, + "step": 2380 + }, + { + "epoch": 0.4592179844365453, + "grad_norm": 1.2821637392044067, + "learning_rate": 0.00028623715551622187, + "loss": 0.7329, + "step": 2390 + }, + { + "epoch": 0.4611393985973677, + "grad_norm": 1.3402514457702637, + "learning_rate": 0.0002847278445854522, + "loss": 0.88, + "step": 2400 + }, + { + "epoch": 0.46306081275819005, + "grad_norm": 1.2902830839157104, + "learning_rate": 0.00028321724186108226, + "loss": 0.927, + "step": 2410 + }, + { + "epoch": 0.4649822269190124, + "grad_norm": 1.1409978866577148, + "learning_rate": 0.00028170540353395694, + "loss": 0.8171, + "step": 2420 + }, + { + "epoch": 0.4669036410798348, + "grad_norm": 1.7634387016296387, + "learning_rate": 0.00028019238584088286, + "loss": 0.8339, + "step": 2430 + }, + { + "epoch": 0.4688250552406571, + "grad_norm": 1.3894178867340088, + "learning_rate": 0.00027867824506253605, + "loss": 0.8551, + "step": 2440 + }, + { + "epoch": 0.47074646940147946, + "grad_norm": 1.396325707435608, + "learning_rate": 0.00027716303752136864, + "loss": 0.8561, + "step": 2450 + }, + { + "epoch": 0.4726678835623018, + "grad_norm": 1.8292694091796875, + "learning_rate": 0.00027564681957951406, + "loss": 0.9106, + "step": 2460 + }, + { + "epoch": 0.4745892977231242, + "grad_norm": 1.632660984992981, + "learning_rate": 0.00027412964763669006, + "loss": 0.7915, + "step": 2470 + }, + { + "epoch": 0.47651071188394656, + "grad_norm": 1.5735293626785278, + "learning_rate": 0.000272611578128101, + "loss": 0.7958, + "step": 2480 + }, + { + "epoch": 0.4784321260447689, + "grad_norm": 1.52776038646698, + "learning_rate": 0.00027109266752233847, + "loss": 0.8285, + "step": 2490 + }, + { + "epoch": 0.4803535402055913, + "grad_norm": 1.4657976627349854, + "learning_rate": 0.0002695729723192811, + "loss": 0.8942, + "step": 2500 + }, + { + "epoch": 0.48227495436641366, + "grad_norm": 2.0572874546051025, + "learning_rate": 0.0002680525490479925, + "loss": 0.8646, + "step": 2510 + }, + { + "epoch": 0.484196368527236, + "grad_norm": 1.147052526473999, + "learning_rate": 0.0002665314542646188, + "loss": 0.8884, + "step": 2520 + }, + { + "epoch": 0.4861177826880584, + "grad_norm": 1.3121823072433472, + "learning_rate": 0.00026500974455028473, + "loss": 0.894, + "step": 2530 + }, + { + "epoch": 0.48803919684888075, + "grad_norm": 2.331324577331543, + "learning_rate": 0.00026348747650898897, + "loss": 0.8319, + "step": 2540 + }, + { + "epoch": 0.4899606110097031, + "grad_norm": 1.9230785369873047, + "learning_rate": 0.0002619647067654988, + "loss": 0.7815, + "step": 2550 + }, + { + "epoch": 0.4918820251705255, + "grad_norm": 0.8924723863601685, + "learning_rate": 0.00026044149196324324, + "loss": 0.8115, + "step": 2560 + }, + { + "epoch": 0.49380343933134785, + "grad_norm": 1.117889165878296, + "learning_rate": 0.00025891788876220706, + "loss": 0.7597, + "step": 2570 + }, + { + "epoch": 0.4957248534921702, + "grad_norm": 0.8507145047187805, + "learning_rate": 0.00025739395383682205, + "loss": 0.765, + "step": 2580 + }, + { + "epoch": 0.4976462676529926, + "grad_norm": 1.1917976140975952, + "learning_rate": 0.00025586974387385947, + "loss": 0.8403, + "step": 2590 + }, + { + "epoch": 0.49956768181381495, + "grad_norm": 1.5689092874526978, + "learning_rate": 0.0002543453155703214, + "loss": 0.8236, + "step": 2600 + }, + { + "epoch": 0.5014890959746373, + "grad_norm": 1.2577379941940308, + "learning_rate": 0.00025282072563133167, + "loss": 0.8568, + "step": 2610 + }, + { + "epoch": 0.5034105101354597, + "grad_norm": 1.9359431266784668, + "learning_rate": 0.0002512960307680266, + "loss": 0.8204, + "step": 2620 + }, + { + "epoch": 0.505331924296282, + "grad_norm": 1.4619979858398438, + "learning_rate": 0.00024977128769544524, + "loss": 0.7493, + "step": 2630 + }, + { + "epoch": 0.5072533384571044, + "grad_norm": 1.4376741647720337, + "learning_rate": 0.00024824655313042014, + "loss": 0.8281, + "step": 2640 + }, + { + "epoch": 0.5091747526179268, + "grad_norm": 1.581264853477478, + "learning_rate": 0.0002467218837894674, + "loss": 0.7787, + "step": 2650 + }, + { + "epoch": 0.5110961667787491, + "grad_norm": 1.4558833837509155, + "learning_rate": 0.0002451973363866766, + "loss": 0.7455, + "step": 2660 + }, + { + "epoch": 0.5130175809395715, + "grad_norm": 1.8191595077514648, + "learning_rate": 0.000243672967631602, + "loss": 0.8622, + "step": 2670 + }, + { + "epoch": 0.5149389951003939, + "grad_norm": 1.5446815490722656, + "learning_rate": 0.00024214883422715212, + "loss": 0.7506, + "step": 2680 + }, + { + "epoch": 0.5168604092612162, + "grad_norm": 1.3518351316452026, + "learning_rate": 0.00024062499286748142, + "loss": 0.7654, + "step": 2690 + }, + { + "epoch": 0.5187818234220386, + "grad_norm": 1.2076045274734497, + "learning_rate": 0.0002391015002358807, + "loss": 0.8667, + "step": 2700 + }, + { + "epoch": 0.520703237582861, + "grad_norm": 1.7851399183273315, + "learning_rate": 0.00023757841300266896, + "loss": 0.8002, + "step": 2710 + }, + { + "epoch": 0.5226246517436833, + "grad_norm": 1.0870471000671387, + "learning_rate": 0.00023605578782308538, + "loss": 0.8146, + "step": 2720 + }, + { + "epoch": 0.5245460659045057, + "grad_norm": 1.481876015663147, + "learning_rate": 0.0002345336813351819, + "loss": 0.8197, + "step": 2730 + }, + { + "epoch": 0.5264674800653281, + "grad_norm": 1.550943374633789, + "learning_rate": 0.00023301215015771607, + "loss": 0.7727, + "step": 2740 + }, + { + "epoch": 0.5283888942261504, + "grad_norm": 1.1842530965805054, + "learning_rate": 0.0002314912508880456, + "loss": 0.7361, + "step": 2750 + }, + { + "epoch": 0.5303103083869728, + "grad_norm": 1.2636481523513794, + "learning_rate": 0.0002299710401000226, + "loss": 0.8723, + "step": 2760 + }, + { + "epoch": 0.5322317225477952, + "grad_norm": 0.6428311467170715, + "learning_rate": 0.0002284515743418893, + "loss": 0.7911, + "step": 2770 + }, + { + "epoch": 0.5341531367086175, + "grad_norm": 1.473262906074524, + "learning_rate": 0.00022693291013417452, + "loss": 0.7811, + "step": 2780 + }, + { + "epoch": 0.5360745508694399, + "grad_norm": 1.4524637460708618, + "learning_rate": 0.00022541510396759134, + "loss": 0.7198, + "step": 2790 + }, + { + "epoch": 0.5379959650302623, + "grad_norm": 2.153787851333618, + "learning_rate": 0.00022389821230093575, + "loss": 0.8221, + "step": 2800 + }, + { + "epoch": 0.5399173791910846, + "grad_norm": 1.4184430837631226, + "learning_rate": 0.00022238229155898656, + "loss": 0.8138, + "step": 2810 + }, + { + "epoch": 0.541838793351907, + "grad_norm": 1.007478952407837, + "learning_rate": 0.00022086739813040622, + "loss": 0.7708, + "step": 2820 + }, + { + "epoch": 0.5437602075127294, + "grad_norm": 1.315148949623108, + "learning_rate": 0.00021935358836564406, + "loss": 0.7335, + "step": 2830 + }, + { + "epoch": 0.5456816216735517, + "grad_norm": 1.2237809896469116, + "learning_rate": 0.0002178409185748392, + "loss": 0.8343, + "step": 2840 + }, + { + "epoch": 0.5476030358343741, + "grad_norm": 1.43268883228302, + "learning_rate": 0.00021632944502572663, + "loss": 0.7771, + "step": 2850 + }, + { + "epoch": 0.5495244499951965, + "grad_norm": 1.8583613634109497, + "learning_rate": 0.000214819223941544, + "loss": 0.8243, + "step": 2860 + }, + { + "epoch": 0.5514458641560188, + "grad_norm": 1.0081727504730225, + "learning_rate": 0.00021331031149894014, + "loss": 0.7548, + "step": 2870 + }, + { + "epoch": 0.5533672783168412, + "grad_norm": 1.6709460020065308, + "learning_rate": 0.0002118027638258856, + "loss": 0.8975, + "step": 2880 + }, + { + "epoch": 0.5552886924776635, + "grad_norm": 1.0278853178024292, + "learning_rate": 0.0002102966369995847, + "loss": 0.7381, + "step": 2890 + }, + { + "epoch": 0.5572101066384859, + "grad_norm": 1.155301570892334, + "learning_rate": 0.00020879198704438945, + "loss": 0.867, + "step": 2900 + }, + { + "epoch": 0.5591315207993083, + "grad_norm": 0.615483283996582, + "learning_rate": 0.0002072888699297162, + "loss": 0.8096, + "step": 2910 + }, + { + "epoch": 0.5610529349601306, + "grad_norm": 1.269416093826294, + "learning_rate": 0.0002057873415679628, + "loss": 0.7184, + "step": 2920 + }, + { + "epoch": 0.562974349120953, + "grad_norm": 1.3008477687835693, + "learning_rate": 0.0002042874578124295, + "loss": 0.834, + "step": 2930 + }, + { + "epoch": 0.5648957632817754, + "grad_norm": 1.5526058673858643, + "learning_rate": 0.00020278927445524104, + "loss": 0.6741, + "step": 2940 + }, + { + "epoch": 0.5668171774425977, + "grad_norm": 1.1507304906845093, + "learning_rate": 0.00020129284722527127, + "loss": 0.7561, + "step": 2950 + }, + { + "epoch": 0.5687385916034201, + "grad_norm": 1.2722505331039429, + "learning_rate": 0.00019979823178607043, + "loss": 0.7212, + "step": 2960 + }, + { + "epoch": 0.5706600057642425, + "grad_norm": 1.5021613836288452, + "learning_rate": 0.00019830548373379425, + "loss": 0.7767, + "step": 2970 + }, + { + "epoch": 0.5725814199250648, + "grad_norm": 1.3983266353607178, + "learning_rate": 0.00019681465859513632, + "loss": 0.7895, + "step": 2980 + }, + { + "epoch": 0.5745028340858872, + "grad_norm": 1.1755023002624512, + "learning_rate": 0.00019532581182526225, + "loss": 0.8512, + "step": 2990 + }, + { + "epoch": 0.5764242482467096, + "grad_norm": 1.3724703788757324, + "learning_rate": 0.00019383899880574696, + "loss": 0.8009, + "step": 3000 + }, + { + "epoch": 0.5783456624075319, + "grad_norm": 1.3599048852920532, + "learning_rate": 0.00019235427484251474, + "loss": 0.828, + "step": 3010 + }, + { + "epoch": 0.5802670765683543, + "grad_norm": 1.1706438064575195, + "learning_rate": 0.00019087169516378202, + "loss": 0.7888, + "step": 3020 + }, + { + "epoch": 0.5821884907291767, + "grad_norm": 1.0189845561981201, + "learning_rate": 0.00018939131491800277, + "loss": 0.7479, + "step": 3030 + }, + { + "epoch": 0.584109904889999, + "grad_norm": 1.106514573097229, + "learning_rate": 0.00018791318917181726, + "loss": 0.8299, + "step": 3040 + }, + { + "epoch": 0.5860313190508214, + "grad_norm": 1.3856253623962402, + "learning_rate": 0.00018643737290800392, + "loss": 0.7903, + "step": 3050 + }, + { + "epoch": 0.5879527332116438, + "grad_norm": 1.2991724014282227, + "learning_rate": 0.0001849639210234337, + "loss": 0.7899, + "step": 3060 + }, + { + "epoch": 0.5898741473724661, + "grad_norm": 1.586128830909729, + "learning_rate": 0.00018349288832702836, + "loss": 0.8614, + "step": 3070 + }, + { + "epoch": 0.5917955615332885, + "grad_norm": 1.5094038248062134, + "learning_rate": 0.00018202432953772147, + "loss": 0.8301, + "step": 3080 + }, + { + "epoch": 0.5937169756941109, + "grad_norm": 1.4626994132995605, + "learning_rate": 0.00018055829928242316, + "loss": 0.7848, + "step": 3090 + }, + { + "epoch": 0.5956383898549332, + "grad_norm": 1.445717215538025, + "learning_rate": 0.00017909485209398817, + "loss": 0.8187, + "step": 3100 + }, + { + "epoch": 0.5975598040157556, + "grad_norm": 1.1802334785461426, + "learning_rate": 0.00017763404240918706, + "loss": 0.8007, + "step": 3110 + }, + { + "epoch": 0.599481218176578, + "grad_norm": 1.464058756828308, + "learning_rate": 0.00017617592456668177, + "loss": 0.7304, + "step": 3120 + }, + { + "epoch": 0.6014026323374003, + "grad_norm": 1.331037163734436, + "learning_rate": 0.0001747205528050039, + "loss": 0.7858, + "step": 3130 + }, + { + "epoch": 0.6033240464982227, + "grad_norm": 0.8612129092216492, + "learning_rate": 0.00017326798126053738, + "loss": 0.7141, + "step": 3140 + }, + { + "epoch": 0.605245460659045, + "grad_norm": 1.2344214916229248, + "learning_rate": 0.00017181826396550477, + "loss": 0.8229, + "step": 3150 + }, + { + "epoch": 0.6071668748198674, + "grad_norm": 1.0575777292251587, + "learning_rate": 0.00017037145484595712, + "loss": 0.6958, + "step": 3160 + }, + { + "epoch": 0.6090882889806898, + "grad_norm": 1.1421902179718018, + "learning_rate": 0.0001689276077197684, + "loss": 0.7866, + "step": 3170 + }, + { + "epoch": 0.6110097031415122, + "grad_norm": 1.1171417236328125, + "learning_rate": 0.00016748677629463331, + "loss": 0.7247, + "step": 3180 + }, + { + "epoch": 0.6129311173023345, + "grad_norm": 0.9187817573547363, + "learning_rate": 0.00016604901416606988, + "loss": 0.7873, + "step": 3190 + }, + { + "epoch": 0.6148525314631569, + "grad_norm": 0.9294849038124084, + "learning_rate": 0.00016461437481542527, + "loss": 0.6915, + "step": 3200 + }, + { + "epoch": 0.6167739456239792, + "grad_norm": 1.6183511018753052, + "learning_rate": 0.00016318291160788678, + "loss": 0.7612, + "step": 3210 + }, + { + "epoch": 0.6186953597848016, + "grad_norm": 1.2286678552627563, + "learning_rate": 0.00016175467779049683, + "loss": 0.788, + "step": 3220 + }, + { + "epoch": 0.620616773945624, + "grad_norm": 1.836376428604126, + "learning_rate": 0.00016032972649017205, + "loss": 0.7777, + "step": 3230 + }, + { + "epoch": 0.6225381881064463, + "grad_norm": 1.4563829898834229, + "learning_rate": 0.00015890811071172717, + "loss": 0.6769, + "step": 3240 + }, + { + "epoch": 0.6244596022672687, + "grad_norm": 1.6083195209503174, + "learning_rate": 0.00015748988333590347, + "loss": 0.8031, + "step": 3250 + }, + { + "epoch": 0.6263810164280911, + "grad_norm": 1.3100241422653198, + "learning_rate": 0.00015607509711740175, + "loss": 0.7412, + "step": 3260 + }, + { + "epoch": 0.6283024305889134, + "grad_norm": 1.613335132598877, + "learning_rate": 0.00015466380468291986, + "loss": 0.6871, + "step": 3270 + }, + { + "epoch": 0.6302238447497358, + "grad_norm": 1.3441022634506226, + "learning_rate": 0.00015325605852919502, + "loss": 0.7887, + "step": 3280 + }, + { + "epoch": 0.6321452589105582, + "grad_norm": 1.4960367679595947, + "learning_rate": 0.00015185191102105134, + "loss": 0.7291, + "step": 3290 + }, + { + "epoch": 0.6340666730713805, + "grad_norm": 1.2021230459213257, + "learning_rate": 0.00015045141438945186, + "loss": 0.7821, + "step": 3300 + }, + { + "epoch": 0.6359880872322029, + "grad_norm": 1.384557843208313, + "learning_rate": 0.00014905462072955548, + "loss": 0.7331, + "step": 3310 + }, + { + "epoch": 0.6379095013930253, + "grad_norm": 1.8842626810073853, + "learning_rate": 0.00014766158199877943, + "loss": 0.7579, + "step": 3320 + }, + { + "epoch": 0.6398309155538476, + "grad_norm": 1.3242464065551758, + "learning_rate": 0.00014627235001486664, + "loss": 0.7771, + "step": 3330 + }, + { + "epoch": 0.64175232971467, + "grad_norm": 1.142120599746704, + "learning_rate": 0.00014488697645395775, + "loss": 0.7195, + "step": 3340 + }, + { + "epoch": 0.6436737438754924, + "grad_norm": 1.052718162536621, + "learning_rate": 0.00014350551284866942, + "loss": 0.7954, + "step": 3350 + }, + { + "epoch": 0.6455951580363147, + "grad_norm": 1.222748041152954, + "learning_rate": 0.00014212801058617714, + "loss": 0.7353, + "step": 3360 + }, + { + "epoch": 0.6475165721971371, + "grad_norm": 1.3705120086669922, + "learning_rate": 0.00014075452090630392, + "loss": 0.7094, + "step": 3370 + }, + { + "epoch": 0.6494379863579595, + "grad_norm": 1.1215436458587646, + "learning_rate": 0.0001393850948996139, + "loss": 0.8098, + "step": 3380 + }, + { + "epoch": 0.6513594005187818, + "grad_norm": 1.4095954895019531, + "learning_rate": 0.00013801978350551242, + "loss": 0.7171, + "step": 3390 + }, + { + "epoch": 0.6532808146796042, + "grad_norm": 0.9342811107635498, + "learning_rate": 0.00013665863751035118, + "loss": 0.7001, + "step": 3400 + }, + { + "epoch": 0.6552022288404266, + "grad_norm": 0.9027429223060608, + "learning_rate": 0.00013530170754553857, + "loss": 0.7636, + "step": 3410 + }, + { + "epoch": 0.6571236430012489, + "grad_norm": 1.1059980392456055, + "learning_rate": 0.00013394904408565682, + "loss": 0.6662, + "step": 3420 + }, + { + "epoch": 0.6590450571620713, + "grad_norm": 1.402596116065979, + "learning_rate": 0.00013260069744658404, + "loss": 0.7067, + "step": 3430 + }, + { + "epoch": 0.6609664713228937, + "grad_norm": 1.4740309715270996, + "learning_rate": 0.00013125671778362307, + "loss": 0.7453, + "step": 3440 + }, + { + "epoch": 0.662887885483716, + "grad_norm": 1.6051650047302246, + "learning_rate": 0.0001299171550896355, + "loss": 0.7181, + "step": 3450 + }, + { + "epoch": 0.6648092996445384, + "grad_norm": 1.1679162979125977, + "learning_rate": 0.00012858205919318206, + "loss": 0.7084, + "step": 3460 + }, + { + "epoch": 0.6667307138053608, + "grad_norm": 1.467795729637146, + "learning_rate": 0.00012725147975666947, + "loss": 0.7222, + "step": 3470 + }, + { + "epoch": 0.6686521279661831, + "grad_norm": 1.1850277185440063, + "learning_rate": 0.00012592546627450242, + "loss": 0.7365, + "step": 3480 + }, + { + "epoch": 0.6705735421270055, + "grad_norm": 1.2745028734207153, + "learning_rate": 0.00012460406807124314, + "loss": 0.7828, + "step": 3490 + }, + { + "epoch": 0.6724949562878278, + "grad_norm": 1.2522541284561157, + "learning_rate": 0.0001232873342997764, + "loss": 0.729, + "step": 3500 + }, + { + "epoch": 0.6744163704486502, + "grad_norm": 1.3559709787368774, + "learning_rate": 0.00012197531393948111, + "loss": 0.7295, + "step": 3510 + }, + { + "epoch": 0.6763377846094726, + "grad_norm": 1.270632028579712, + "learning_rate": 0.00012066805579440849, + "loss": 0.6994, + "step": 3520 + }, + { + "epoch": 0.678259198770295, + "grad_norm": 1.3047348260879517, + "learning_rate": 0.00011936560849146657, + "loss": 0.6652, + "step": 3530 + }, + { + "epoch": 0.6801806129311173, + "grad_norm": 1.5254732370376587, + "learning_rate": 0.00011806802047861148, + "loss": 0.7909, + "step": 3540 + }, + { + "epoch": 0.6821020270919397, + "grad_norm": 1.30349600315094, + "learning_rate": 0.00011677534002304529, + "loss": 0.7628, + "step": 3550 + }, + { + "epoch": 0.684023441252762, + "grad_norm": 1.0586496591567993, + "learning_rate": 0.00011548761520942048, + "loss": 0.6931, + "step": 3560 + }, + { + "epoch": 0.6859448554135844, + "grad_norm": 1.3567843437194824, + "learning_rate": 0.00011420489393805142, + "loss": 0.6769, + "step": 3570 + }, + { + "epoch": 0.6878662695744068, + "grad_norm": 1.2144553661346436, + "learning_rate": 0.0001129272239231327, + "loss": 0.711, + "step": 3580 + }, + { + "epoch": 0.6897876837352291, + "grad_norm": 1.1030033826828003, + "learning_rate": 0.00011165465269096375, + "loss": 0.7395, + "step": 3590 + }, + { + "epoch": 0.6917090978960515, + "grad_norm": 1.145440697669983, + "learning_rate": 0.00011038722757818163, + "loss": 0.6487, + "step": 3600 + }, + { + "epoch": 0.6936305120568739, + "grad_norm": 1.0338114500045776, + "learning_rate": 0.00010912499573000006, + "loss": 0.6389, + "step": 3610 + }, + { + "epoch": 0.6955519262176962, + "grad_norm": 1.4889144897460938, + "learning_rate": 0.00010786800409845537, + "loss": 0.7449, + "step": 3620 + }, + { + "epoch": 0.6974733403785186, + "grad_norm": 1.3305447101593018, + "learning_rate": 0.00010661629944066043, + "loss": 0.6682, + "step": 3630 + }, + { + "epoch": 0.699394754539341, + "grad_norm": 0.9120017886161804, + "learning_rate": 0.0001053699283170649, + "loss": 0.7316, + "step": 3640 + }, + { + "epoch": 0.7013161687001633, + "grad_norm": 0.8771267533302307, + "learning_rate": 0.00010412893708972387, + "loss": 0.726, + "step": 3650 + }, + { + "epoch": 0.7032375828609857, + "grad_norm": 0.9893380999565125, + "learning_rate": 0.00010289337192057296, + "loss": 0.6199, + "step": 3660 + }, + { + "epoch": 0.7051589970218081, + "grad_norm": 1.0728754997253418, + "learning_rate": 0.00010166327876971126, + "loss": 0.7224, + "step": 3670 + }, + { + "epoch": 0.7070804111826304, + "grad_norm": 0.4639236330986023, + "learning_rate": 0.00010043870339369174, + "loss": 0.7264, + "step": 3680 + }, + { + "epoch": 0.7090018253434528, + "grad_norm": 1.6407727003097534, + "learning_rate": 9.92196913438192e-05, + "loss": 0.7552, + "step": 3690 + }, + { + "epoch": 0.7109232395042752, + "grad_norm": 1.1802722215652466, + "learning_rate": 9.800628796445585e-05, + "loss": 0.7481, + "step": 3700 + }, + { + "epoch": 0.7128446536650975, + "grad_norm": 1.6073251962661743, + "learning_rate": 9.67985383913347e-05, + "loss": 0.7225, + "step": 3710 + }, + { + "epoch": 0.7147660678259199, + "grad_norm": 1.8788129091262817, + "learning_rate": 9.559648754988054e-05, + "loss": 0.6767, + "step": 3720 + }, + { + "epoch": 0.7166874819867423, + "grad_norm": 1.3577216863632202, + "learning_rate": 9.440018015353874e-05, + "loss": 0.722, + "step": 3730 + }, + { + "epoch": 0.7186088961475646, + "grad_norm": 1.0307360887527466, + "learning_rate": 9.320966070211226e-05, + "loss": 0.7395, + "step": 3740 + }, + { + "epoch": 0.720530310308387, + "grad_norm": 1.6263340711593628, + "learning_rate": 9.202497348010608e-05, + "loss": 0.7294, + "step": 3750 + }, + { + "epoch": 0.7224517244692094, + "grad_norm": 0.6837071776390076, + "learning_rate": 9.084616255508013e-05, + "loss": 0.6566, + "step": 3760 + }, + { + "epoch": 0.7243731386300317, + "grad_norm": 0.9681917428970337, + "learning_rate": 8.967327177600997e-05, + "loss": 0.7113, + "step": 3770 + }, + { + "epoch": 0.7262945527908541, + "grad_norm": 1.1300331354141235, + "learning_rate": 8.850634477165581e-05, + "loss": 0.7455, + "step": 3780 + }, + { + "epoch": 0.7282159669516765, + "grad_norm": 1.0961904525756836, + "learning_rate": 8.734542494893955e-05, + "loss": 0.6819, + "step": 3790 + }, + { + "epoch": 0.7301373811124988, + "grad_norm": 0.8156876564025879, + "learning_rate": 8.619055549132992e-05, + "loss": 0.6541, + "step": 3800 + }, + { + "epoch": 0.7320587952733212, + "grad_norm": 1.1605948209762573, + "learning_rate": 8.504177935723672e-05, + "loss": 0.6239, + "step": 3810 + }, + { + "epoch": 0.7339802094341435, + "grad_norm": 1.2597169876098633, + "learning_rate": 8.389913927841231e-05, + "loss": 0.6716, + "step": 3820 + }, + { + "epoch": 0.7359016235949659, + "grad_norm": 0.7605863213539124, + "learning_rate": 8.276267775836266e-05, + "loss": 0.6895, + "step": 3830 + }, + { + "epoch": 0.7378230377557883, + "grad_norm": 1.0140622854232788, + "learning_rate": 8.163243707076548e-05, + "loss": 0.64, + "step": 3840 + }, + { + "epoch": 0.7397444519166106, + "grad_norm": 0.9782150387763977, + "learning_rate": 8.050845925789862e-05, + "loss": 0.6633, + "step": 3850 + }, + { + "epoch": 0.741665866077433, + "grad_norm": 1.9944454431533813, + "learning_rate": 7.939078612907567e-05, + "loss": 0.7092, + "step": 3860 + }, + { + "epoch": 0.7435872802382554, + "grad_norm": 0.9737130403518677, + "learning_rate": 7.827945925909094e-05, + "loss": 0.6458, + "step": 3870 + }, + { + "epoch": 0.7455086943990777, + "grad_norm": 0.9813506007194519, + "learning_rate": 7.71745199866729e-05, + "loss": 0.6435, + "step": 3880 + }, + { + "epoch": 0.7474301085599001, + "grad_norm": 0.6654618978500366, + "learning_rate": 7.607600941294657e-05, + "loss": 0.6256, + "step": 3890 + }, + { + "epoch": 0.7493515227207225, + "grad_norm": 1.6668603420257568, + "learning_rate": 7.498396839990456e-05, + "loss": 0.7146, + "step": 3900 + }, + { + "epoch": 0.7512729368815448, + "grad_norm": 1.2132741212844849, + "learning_rate": 7.389843756888712e-05, + "loss": 0.6987, + "step": 3910 + }, + { + "epoch": 0.7531943510423672, + "grad_norm": 0.9006101489067078, + "learning_rate": 7.281945729907119e-05, + "loss": 0.6177, + "step": 3920 + }, + { + "epoch": 0.7551157652031896, + "grad_norm": 1.2604845762252808, + "learning_rate": 7.174706772596831e-05, + "loss": 0.6868, + "step": 3930 + }, + { + "epoch": 0.7570371793640119, + "grad_norm": 1.1921509504318237, + "learning_rate": 7.068130873993181e-05, + "loss": 0.6461, + "step": 3940 + }, + { + "epoch": 0.7589585935248343, + "grad_norm": 0.9460396766662598, + "learning_rate": 6.962221998467261e-05, + "loss": 0.7165, + "step": 3950 + }, + { + "epoch": 0.7608800076856567, + "grad_norm": 1.009789228439331, + "learning_rate": 6.8569840855785e-05, + "loss": 0.7098, + "step": 3960 + }, + { + "epoch": 0.762801421846479, + "grad_norm": 1.3344640731811523, + "learning_rate": 6.752421049928123e-05, + "loss": 0.6595, + "step": 3970 + }, + { + "epoch": 0.7647228360073014, + "grad_norm": 1.0439685583114624, + "learning_rate": 6.648536781013495e-05, + "loss": 0.6576, + "step": 3980 + }, + { + "epoch": 0.7666442501681238, + "grad_norm": 0.7947928309440613, + "learning_rate": 6.545335143083483e-05, + "loss": 0.7006, + "step": 3990 + }, + { + "epoch": 0.7685656643289461, + "grad_norm": 1.037701964378357, + "learning_rate": 6.442819974994669e-05, + "loss": 0.7077, + "step": 4000 + }, + { + "epoch": 0.7704870784897685, + "grad_norm": 1.210694670677185, + "learning_rate": 6.34099509006861e-05, + "loss": 0.7446, + "step": 4010 + }, + { + "epoch": 0.7724084926505909, + "grad_norm": 1.6502310037612915, + "learning_rate": 6.239864275949958e-05, + "loss": 0.6645, + "step": 4020 + }, + { + "epoch": 0.7743299068114132, + "grad_norm": 0.899091362953186, + "learning_rate": 6.139431294465558e-05, + "loss": 0.693, + "step": 4030 + }, + { + "epoch": 0.7762513209722356, + "grad_norm": 1.1899176836013794, + "learning_rate": 6.03969988148457e-05, + "loss": 0.7106, + "step": 4040 + }, + { + "epoch": 0.778172735133058, + "grad_norm": 0.8680855631828308, + "learning_rate": 5.940673746779421e-05, + "loss": 0.6068, + "step": 4050 + }, + { + "epoch": 0.7800941492938803, + "grad_norm": 0.9416540265083313, + "learning_rate": 5.842356573887886e-05, + "loss": 0.6198, + "step": 4060 + }, + { + "epoch": 0.7820155634547027, + "grad_norm": 1.4808692932128906, + "learning_rate": 5.744752019976027e-05, + "loss": 0.6638, + "step": 4070 + }, + { + "epoch": 0.783936977615525, + "grad_norm": 0.8431961536407471, + "learning_rate": 5.647863715702173e-05, + "loss": 0.6449, + "step": 4080 + }, + { + "epoch": 0.7858583917763474, + "grad_norm": 0.9810335636138916, + "learning_rate": 5.5516952650818605e-05, + "loss": 0.6363, + "step": 4090 + }, + { + "epoch": 0.7877798059371698, + "grad_norm": 1.1404001712799072, + "learning_rate": 5.45625024535377e-05, + "loss": 0.6726, + "step": 4100 + }, + { + "epoch": 0.7897012200979922, + "grad_norm": 1.1580007076263428, + "learning_rate": 5.36153220684667e-05, + "loss": 0.6647, + "step": 4110 + }, + { + "epoch": 0.7916226342588145, + "grad_norm": 1.1604604721069336, + "learning_rate": 5.267544672847352e-05, + "loss": 0.7539, + "step": 4120 + }, + { + "epoch": 0.7935440484196369, + "grad_norm": 0.8880569338798523, + "learning_rate": 5.174291139469559e-05, + "loss": 0.6777, + "step": 4130 + }, + { + "epoch": 0.7954654625804592, + "grad_norm": 1.15243399143219, + "learning_rate": 5.081775075523959e-05, + "loss": 0.6044, + "step": 4140 + }, + { + "epoch": 0.7973868767412816, + "grad_norm": 1.313981056213379, + "learning_rate": 4.989999922389102e-05, + "loss": 0.6544, + "step": 4150 + }, + { + "epoch": 0.799308290902104, + "grad_norm": 1.1018239259719849, + "learning_rate": 4.898969093883396e-05, + "loss": 0.6708, + "step": 4160 + }, + { + "epoch": 0.8012297050629263, + "grad_norm": 1.0654897689819336, + "learning_rate": 4.808685976138147e-05, + "loss": 0.7295, + "step": 4170 + }, + { + "epoch": 0.8031511192237487, + "grad_norm": 0.6468730568885803, + "learning_rate": 4.719153927471598e-05, + "loss": 0.6127, + "step": 4180 + }, + { + "epoch": 0.8050725333845711, + "grad_norm": 0.9620100259780884, + "learning_rate": 4.6303762782639895e-05, + "loss": 0.6429, + "step": 4190 + }, + { + "epoch": 0.8069939475453934, + "grad_norm": 1.0371164083480835, + "learning_rate": 4.542356330833697e-05, + "loss": 0.6625, + "step": 4200 + }, + { + "epoch": 0.8089153617062158, + "grad_norm": 1.0234382152557373, + "learning_rate": 4.455097359314361e-05, + "loss": 0.6418, + "step": 4210 + }, + { + "epoch": 0.8108367758670382, + "grad_norm": 1.1065011024475098, + "learning_rate": 4.368602609533146e-05, + "loss": 0.6692, + "step": 4220 + }, + { + "epoch": 0.8127581900278605, + "grad_norm": 0.9469298124313354, + "learning_rate": 4.282875298889966e-05, + "loss": 0.6272, + "step": 4230 + }, + { + "epoch": 0.8146796041886829, + "grad_norm": 0.7343427538871765, + "learning_rate": 4.1979186162378115e-05, + "loss": 0.6458, + "step": 4240 + }, + { + "epoch": 0.8166010183495053, + "grad_norm": 1.0217844247817993, + "learning_rate": 4.113735721764161e-05, + "loss": 0.6489, + "step": 4250 + }, + { + "epoch": 0.8185224325103276, + "grad_norm": 0.8793905377388, + "learning_rate": 4.030329746873365e-05, + "loss": 0.6778, + "step": 4260 + }, + { + "epoch": 0.82044384667115, + "grad_norm": 1.3632718324661255, + "learning_rate": 3.9477037940702346e-05, + "loss": 0.6279, + "step": 4270 + }, + { + "epoch": 0.8223652608319724, + "grad_norm": 0.9788404703140259, + "learning_rate": 3.865860936844595e-05, + "loss": 0.6902, + "step": 4280 + }, + { + "epoch": 0.8242866749927947, + "grad_norm": 0.5577342510223389, + "learning_rate": 3.7848042195569725e-05, + "loss": 0.5732, + "step": 4290 + }, + { + "epoch": 0.8262080891536171, + "grad_norm": 1.5902769565582275, + "learning_rate": 3.704536657325347e-05, + "loss": 0.6999, + "step": 4300 + }, + { + "epoch": 0.8281295033144395, + "grad_norm": 1.079472303390503, + "learning_rate": 3.625061235913002e-05, + "loss": 0.6913, + "step": 4310 + }, + { + "epoch": 0.8300509174752618, + "grad_norm": 1.0420955419540405, + "learning_rate": 3.5463809116174555e-05, + "loss": 0.632, + "step": 4320 + }, + { + "epoch": 0.8319723316360842, + "grad_norm": 0.9489886164665222, + "learning_rate": 3.468498611160495e-05, + "loss": 0.6202, + "step": 4330 + }, + { + "epoch": 0.8338937457969066, + "grad_norm": 1.1345503330230713, + "learning_rate": 3.391417231579308e-05, + "loss": 0.6264, + "step": 4340 + }, + { + "epoch": 0.8358151599577289, + "grad_norm": 1.043662667274475, + "learning_rate": 3.315139640118728e-05, + "loss": 0.5818, + "step": 4350 + }, + { + "epoch": 0.8377365741185513, + "grad_norm": 1.3796449899673462, + "learning_rate": 3.2396686741245765e-05, + "loss": 0.6731, + "step": 4360 + }, + { + "epoch": 0.8396579882793737, + "grad_norm": 0.87774258852005, + "learning_rate": 3.165007140938095e-05, + "loss": 0.7172, + "step": 4370 + }, + { + "epoch": 0.841579402440196, + "grad_norm": 0.8624753952026367, + "learning_rate": 3.0911578177915595e-05, + "loss": 0.6313, + "step": 4380 + }, + { + "epoch": 0.8435008166010184, + "grad_norm": 0.7553603053092957, + "learning_rate": 3.0181234517049654e-05, + "loss": 0.5941, + "step": 4390 + }, + { + "epoch": 0.8454222307618408, + "grad_norm": 1.1863903999328613, + "learning_rate": 2.945906759383815e-05, + "loss": 0.6725, + "step": 4400 + }, + { + "epoch": 0.8473436449226631, + "grad_norm": 0.886174201965332, + "learning_rate": 2.8745104271180933e-05, + "loss": 0.6407, + "step": 4410 + }, + { + "epoch": 0.8492650590834855, + "grad_norm": 0.7463008165359497, + "learning_rate": 2.8039371106823196e-05, + "loss": 0.577, + "step": 4420 + }, + { + "epoch": 0.8511864732443079, + "grad_norm": 1.0898914337158203, + "learning_rate": 2.734189435236789e-05, + "loss": 0.6662, + "step": 4430 + }, + { + "epoch": 0.8531078874051302, + "grad_norm": 1.42017662525177, + "learning_rate": 2.6652699952298994e-05, + "loss": 0.6746, + "step": 4440 + }, + { + "epoch": 0.8550293015659526, + "grad_norm": 0.884002149105072, + "learning_rate": 2.5971813543016475e-05, + "loss": 0.6162, + "step": 4450 + }, + { + "epoch": 0.856950715726775, + "grad_norm": 1.4128063917160034, + "learning_rate": 2.529926045188291e-05, + "loss": 0.6601, + "step": 4460 + }, + { + "epoch": 0.8588721298875973, + "grad_norm": 0.7436163425445557, + "learning_rate": 2.463506569628085e-05, + "loss": 0.6425, + "step": 4470 + }, + { + "epoch": 0.8607935440484197, + "grad_norm": 1.0224891901016235, + "learning_rate": 2.39792539826828e-05, + "loss": 0.6066, + "step": 4480 + }, + { + "epoch": 0.862714958209242, + "grad_norm": 1.1273442506790161, + "learning_rate": 2.3331849705731876e-05, + "loss": 0.5953, + "step": 4490 + }, + { + "epoch": 0.8646363723700644, + "grad_norm": 1.1450196504592896, + "learning_rate": 2.2692876947334406e-05, + "loss": 0.6013, + "step": 4500 + }, + { + "epoch": 0.8665577865308868, + "grad_norm": 1.3181530237197876, + "learning_rate": 2.2062359475764266e-05, + "loss": 0.67, + "step": 4510 + }, + { + "epoch": 0.8684792006917091, + "grad_norm": 1.0618128776550293, + "learning_rate": 2.144032074477861e-05, + "loss": 0.6158, + "step": 4520 + }, + { + "epoch": 0.8704006148525315, + "grad_norm": 0.7197682857513428, + "learning_rate": 2.0826783892745617e-05, + "loss": 0.5484, + "step": 4530 + }, + { + "epoch": 0.8723220290133539, + "grad_norm": 1.1397417783737183, + "learning_rate": 2.0221771741783578e-05, + "loss": 0.5875, + "step": 4540 + }, + { + "epoch": 0.8742434431741762, + "grad_norm": 1.1843700408935547, + "learning_rate": 1.9625306796912158e-05, + "loss": 0.6341, + "step": 4550 + }, + { + "epoch": 0.8761648573349986, + "grad_norm": 1.336387038230896, + "learning_rate": 1.903741124521516e-05, + "loss": 0.6003, + "step": 4560 + }, + { + "epoch": 0.878086271495821, + "grad_norm": 0.7944359183311462, + "learning_rate": 1.8458106955015318e-05, + "loss": 0.6365, + "step": 4570 + }, + { + "epoch": 0.8800076856566433, + "grad_norm": 0.6551217436790466, + "learning_rate": 1.7887415475060646e-05, + "loss": 0.6544, + "step": 4580 + }, + { + "epoch": 0.8819290998174657, + "grad_norm": 1.1356467008590698, + "learning_rate": 1.7325358033723092e-05, + "loss": 0.7436, + "step": 4590 + }, + { + "epoch": 0.8838505139782881, + "grad_norm": 0.9559223651885986, + "learning_rate": 1.6771955538208843e-05, + "loss": 0.6652, + "step": 4600 + }, + { + "epoch": 0.8857719281391104, + "grad_norm": 1.0125079154968262, + "learning_rate": 1.622722857378056e-05, + "loss": 0.6105, + "step": 4610 + }, + { + "epoch": 0.8876933422999328, + "grad_norm": 0.899423360824585, + "learning_rate": 1.5691197402991684e-05, + "loss": 0.5898, + "step": 4620 + }, + { + "epoch": 0.8896147564607552, + "grad_norm": 1.1466995477676392, + "learning_rate": 1.5163881964932653e-05, + "loss": 0.6615, + "step": 4630 + }, + { + "epoch": 0.8915361706215775, + "grad_norm": 1.5939006805419922, + "learning_rate": 1.4645301874489342e-05, + "loss": 0.7415, + "step": 4640 + }, + { + "epoch": 0.8934575847823999, + "grad_norm": 0.6204648613929749, + "learning_rate": 1.4135476421613419e-05, + "loss": 0.6184, + "step": 4650 + }, + { + "epoch": 0.8953789989432223, + "grad_norm": 1.0177160501480103, + "learning_rate": 1.3634424570604682e-05, + "loss": 0.6398, + "step": 4660 + }, + { + "epoch": 0.8973004131040446, + "grad_norm": 1.6055784225463867, + "learning_rate": 1.3142164959405817e-05, + "loss": 0.6313, + "step": 4670 + }, + { + "epoch": 0.899221827264867, + "grad_norm": 1.299264669418335, + "learning_rate": 1.265871589890885e-05, + "loss": 0.7217, + "step": 4680 + }, + { + "epoch": 0.9011432414256894, + "grad_norm": 1.453226089477539, + "learning_rate": 1.2184095372274301e-05, + "loss": 0.6966, + "step": 4690 + }, + { + "epoch": 0.9030646555865117, + "grad_norm": 1.0507123470306396, + "learning_rate": 1.1718321034262125e-05, + "loss": 0.6368, + "step": 4700 + }, + { + "epoch": 0.9049860697473341, + "grad_norm": 0.8116253614425659, + "learning_rate": 1.1261410210574918e-05, + "loss": 0.6051, + "step": 4710 + }, + { + "epoch": 0.9069074839081565, + "grad_norm": 1.0800602436065674, + "learning_rate": 1.0813379897213593e-05, + "loss": 0.628, + "step": 4720 + }, + { + "epoch": 0.9088288980689788, + "grad_norm": 1.0864616632461548, + "learning_rate": 1.0374246759845134e-05, + "loss": 0.5293, + "step": 4730 + }, + { + "epoch": 0.9107503122298012, + "grad_norm": 1.0752801895141602, + "learning_rate": 9.94402713318257e-06, + "loss": 0.6651, + "step": 4740 + }, + { + "epoch": 0.9126717263906235, + "grad_norm": 1.0369071960449219, + "learning_rate": 9.52273702037748e-06, + "loss": 0.6574, + "step": 4750 + }, + { + "epoch": 0.9145931405514459, + "grad_norm": 1.087040662765503, + "learning_rate": 9.110392092424647e-06, + "loss": 0.6523, + "step": 4760 + }, + { + "epoch": 0.9165145547122683, + "grad_norm": 0.632030189037323, + "learning_rate": 8.707007687579177e-06, + "loss": 0.6064, + "step": 4770 + }, + { + "epoch": 0.9184359688730906, + "grad_norm": 1.062684178352356, + "learning_rate": 8.312598810785943e-06, + "loss": 0.6736, + "step": 4780 + }, + { + "epoch": 0.920357383033913, + "grad_norm": 1.3037798404693604, + "learning_rate": 7.927180133121298e-06, + "loss": 0.6999, + "step": 4790 + }, + { + "epoch": 0.9222787971947354, + "grad_norm": 0.8168327808380127, + "learning_rate": 7.550765991247654e-06, + "loss": 0.5788, + "step": 4800 + }, + { + "epoch": 0.9242002113555577, + "grad_norm": 0.6602869629859924, + "learning_rate": 7.183370386879884e-06, + "loss": 0.6263, + "step": 4810 + }, + { + "epoch": 0.9261216255163801, + "grad_norm": 0.7275961637496948, + "learning_rate": 6.825006986264703e-06, + "loss": 0.6584, + "step": 4820 + }, + { + "epoch": 0.9280430396772025, + "grad_norm": 1.1119617223739624, + "learning_rate": 6.475689119672168e-06, + "loss": 0.6648, + "step": 4830 + }, + { + "epoch": 0.9299644538380248, + "grad_norm": 0.8867122530937195, + "learning_rate": 6.135429780899926e-06, + "loss": 0.5792, + "step": 4840 + }, + { + "epoch": 0.9318858679988472, + "grad_norm": 1.2685691118240356, + "learning_rate": 5.804241626789747e-06, + "loss": 0.5848, + "step": 4850 + }, + { + "epoch": 0.9338072821596696, + "grad_norm": 1.0925408601760864, + "learning_rate": 5.482136976756952e-06, + "loss": 0.6466, + "step": 4860 + }, + { + "epoch": 0.9357286963204919, + "grad_norm": 0.6285922527313232, + "learning_rate": 5.169127812331892e-06, + "loss": 0.6179, + "step": 4870 + }, + { + "epoch": 0.9376501104813142, + "grad_norm": 2.0445570945739746, + "learning_rate": 4.865225776714471e-06, + "loss": 0.7558, + "step": 4880 + }, + { + "epoch": 0.9395715246421366, + "grad_norm": 0.9207888245582581, + "learning_rate": 4.570442174340883e-06, + "loss": 0.6298, + "step": 4890 + }, + { + "epoch": 0.9414929388029589, + "grad_norm": 0.5049605369567871, + "learning_rate": 4.284787970463277e-06, + "loss": 0.5918, + "step": 4900 + }, + { + "epoch": 0.9434143529637813, + "grad_norm": 0.7330450415611267, + "learning_rate": 4.008273790741701e-06, + "loss": 0.5832, + "step": 4910 + }, + { + "epoch": 0.9453357671246037, + "grad_norm": 1.1495826244354248, + "learning_rate": 3.7409099208490506e-06, + "loss": 0.6474, + "step": 4920 + }, + { + "epoch": 0.947257181285426, + "grad_norm": 1.0758661031723022, + "learning_rate": 3.4827063060882404e-06, + "loss": 0.6798, + "step": 4930 + }, + { + "epoch": 0.9491785954462484, + "grad_norm": 0.7488206624984741, + "learning_rate": 3.2336725510224986e-06, + "loss": 0.5957, + "step": 4940 + }, + { + "epoch": 0.9511000096070708, + "grad_norm": 0.78911954164505, + "learning_rate": 2.993817919117875e-06, + "loss": 0.6642, + "step": 4950 + }, + { + "epoch": 0.9530214237678931, + "grad_norm": 0.4790767729282379, + "learning_rate": 2.7631513323988777e-06, + "loss": 0.6532, + "step": 4960 + }, + { + "epoch": 0.9549428379287155, + "grad_norm": 1.1064733266830444, + "learning_rate": 2.5416813711163777e-06, + "loss": 0.7227, + "step": 4970 + }, + { + "epoch": 0.9568642520895378, + "grad_norm": 1.0712919235229492, + "learning_rate": 2.329416273428614e-06, + "loss": 0.6789, + "step": 4980 + }, + { + "epoch": 0.9587856662503602, + "grad_norm": 0.9066910147666931, + "learning_rate": 2.1263639350946884e-06, + "loss": 0.6153, + "step": 4990 + }, + { + "epoch": 0.9607070804111826, + "grad_norm": 0.6191006898880005, + "learning_rate": 1.9325319091808845e-06, + "loss": 0.7482, + "step": 5000 + }, + { + "epoch": 0.962628494572005, + "grad_norm": 0.6870371103286743, + "learning_rate": 1.7479274057796146e-06, + "loss": 0.5674, + "step": 5010 + }, + { + "epoch": 0.9645499087328273, + "grad_norm": 1.2739887237548828, + "learning_rate": 1.572557291741411e-06, + "loss": 0.6202, + "step": 5020 + }, + { + "epoch": 0.9664713228936497, + "grad_norm": 0.7925472259521484, + "learning_rate": 1.4064280904192983e-06, + "loss": 0.6017, + "step": 5030 + }, + { + "epoch": 0.968392737054472, + "grad_norm": 1.0697252750396729, + "learning_rate": 1.2495459814262366e-06, + "loss": 0.5934, + "step": 5040 + }, + { + "epoch": 0.9703141512152944, + "grad_norm": 0.6690041422843933, + "learning_rate": 1.101916800405306e-06, + "loss": 0.5856, + "step": 5050 + }, + { + "epoch": 0.9722355653761168, + "grad_norm": 1.0088361501693726, + "learning_rate": 9.635460388124629e-07, + "loss": 0.525, + "step": 5060 + }, + { + "epoch": 0.9741569795369391, + "grad_norm": 1.1077866554260254, + "learning_rate": 8.344388437125372e-07, + "loss": 0.6342, + "step": 5070 + }, + { + "epoch": 0.9760783936977615, + "grad_norm": 0.7781296968460083, + "learning_rate": 7.146000175874412e-07, + "loss": 0.6074, + "step": 5080 + }, + { + "epoch": 0.9779998078585839, + "grad_norm": 1.0012695789337158, + "learning_rate": 6.040340181578119e-07, + "loss": 0.624, + "step": 5090 + }, + { + "epoch": 0.9799212220194062, + "grad_norm": 0.6960607767105103, + "learning_rate": 5.027449582170884e-07, + "loss": 0.6221, + "step": 5100 + }, + { + "epoch": 0.9818426361802286, + "grad_norm": 1.2364157438278198, + "learning_rate": 4.107366054784956e-07, + "loss": 0.7251, + "step": 5110 + }, + { + "epoch": 0.983764050341051, + "grad_norm": 1.0493301153182983, + "learning_rate": 3.28012382434878e-07, + "loss": 0.6961, + "step": 5120 + }, + { + "epoch": 0.9856854645018733, + "grad_norm": 0.7855087518692017, + "learning_rate": 2.54575366231552e-07, + "loss": 0.545, + "step": 5130 + }, + { + "epoch": 0.9876068786626957, + "grad_norm": 0.9644796848297119, + "learning_rate": 1.9042828855159177e-07, + "loss": 0.6529, + "step": 5140 + }, + { + "epoch": 0.9895282928235181, + "grad_norm": 1.0050861835479736, + "learning_rate": 1.3557353551446605e-07, + "loss": 0.6603, + "step": 5150 + }, + { + "epoch": 0.9914497069843404, + "grad_norm": 1.5977165699005127, + "learning_rate": 9.001314758708135e-08, + "loss": 0.6425, + "step": 5160 + }, + { + "epoch": 0.9933711211451628, + "grad_norm": 1.4106450080871582, + "learning_rate": 5.374881950803712e-08, + "loss": 0.6156, + "step": 5170 + }, + { + "epoch": 0.9952925353059852, + "grad_norm": 1.0265008211135864, + "learning_rate": 2.6781900224481792e-08, + "loss": 0.6507, + "step": 5180 + }, + { + "epoch": 0.9972139494668075, + "grad_norm": 1.0650451183319092, + "learning_rate": 9.113392841958445e-09, + "loss": 0.6396, + "step": 5190 + }, + { + "epoch": 0.9991353636276299, + "grad_norm": 0.8263037204742432, + "learning_rate": 7.439545870735476e-10, + "loss": 0.6519, + "step": 5200 + }, + { + "epoch": 0.9999039292919589, + "step": 5204, + "total_flos": 4.294943192106664e+18, + "train_loss": 0.8275534134475567, + "train_runtime": 12276.2585, + "train_samples_per_second": 27.131, + "train_steps_per_second": 0.424 + } + ], + "logging_steps": 10, + "max_steps": 5204, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.294943192106664e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/Ins/indices.json b/Ins/indices.json new file mode 100644 index 0000000000000000000000000000000000000000..43692aa57b0b8e67821d5d1a8dc74df447577097 --- /dev/null +++ b/Ins/indices.json @@ -0,0 +1 @@ +{"0": ["", "", "", ""], "1": ["", "", "", ""], "2": ["", "", "", ""], "3": ["", "", "", ""], "4": ["", "", "", ""], "5": ["", "", "", ""], "6": ["", "", "", ""], "7": ["", "", "", ""], "8": ["", "", "", ""], "9": ["", "", "", ""], "10": ["", "", "", ""], "11": ["", "", "", ""], "12": ["", "", "", ""], "13": ["", "", "", ""], "14": ["", "", "", ""], "15": ["", "", "", ""], "16": ["", "", "", ""], "17": ["", "", "", ""], "18": ["", "", "", ""], "19": ["", "", "", ""], "20": ["", "", "", ""], "21": ["", "", "", ""], "22": ["", "", "", ""], "23": ["", "", "", ""], "24": ["", "", "", ""], "25": ["", "", "", ""], "26": ["", "", "", ""], "27": ["", "", "", ""], "28": ["", "", "", ""], "29": ["", "", "", ""], "30": ["", "", "", ""], "31": ["", "", "", ""], "32": ["", "", "", ""], "33": ["", "", "", ""], "34": ["", "", "", ""], "35": ["", "", "", ""], "36": ["", "", "", ""], "37": ["", "", "", ""], "38": ["", "", "", ""], "39": ["", "", "", ""], "40": ["", "", "", ""], "41": ["", "", "", ""], "42": ["", "", "", ""], "43": ["", "", "", ""], "44": ["", "", "", ""], "45": ["", "", "", ""], "46": ["", "", "", ""], "47": ["", "", "", ""], "48": ["", "", "", ""], "49": ["", "", "", ""], "50": ["", "", "", ""], "51": ["", "", "", ""], "52": ["", "", "", ""], "53": ["", "", "", ""], "54": ["", "", "", ""], "55": ["", "", "", ""], "56": ["", "", "", ""], "57": ["", "", "", ""], "58": ["", "", "", ""], "59": ["", "", "", ""], "60": ["", "", "", ""], "61": ["", "", "", ""], "62": ["", "", "", ""], "63": ["", "", "", ""], "64": ["", "", "", ""], "65": ["", "", "", ""], "66": ["", "", "", ""], "67": ["", "", "", ""], "68": ["", "", "", ""], "69": ["", "", "", ""], "70": ["", "", "", ""], "71": ["", "", "", ""], "72": ["", "", "", ""], "73": ["", "", "", ""], "74": ["", "", "", ""], "75": ["", "", "", ""], "76": ["", "", "", ""], "77": ["", "", "", ""], "78": ["", "", "", ""], "79": ["", "", "", ""], "80": ["", "", "", ""], "81": ["", "", "", ""], "82": ["", "", "", ""], "83": ["", "", "", ""], "84": ["", "", "", ""], "85": ["", "", "", ""], "86": ["", "", "", ""], "87": ["", "", "", ""], "88": ["", "", "", ""], "89": ["", "", "", ""], "90": ["", "", "", ""], "91": ["", "", "", ""], "92": ["", "", "", ""], "93": ["", "", "", ""], "94": ["", "", "", ""], "95": ["", "", "", ""], "96": ["", "", "", ""], "97": ["", "", "", ""], "98": ["", "", "", ""], "99": ["", "", "", ""], "100": ["", "", "", ""], "101": ["", "", "", ""], "102": ["", "", "", ""], "103": ["", "", "", ""], "104": ["", "", "", ""], "105": ["", "", "", ""], "106": ["", "", "", ""], "107": ["", "", "", ""], "108": ["", "", "", ""], "109": ["", "", "", ""], "110": ["", "", "", ""], "111": ["", "", "", ""], "112": ["", "", "", ""], "113": ["", "", "", ""], "114": ["", "", "", ""], "115": ["", "", "", ""], "116": ["", "", "", ""], "117": ["", "", "", ""], "118": ["", "", "", ""], "119": ["", "", "", ""], "120": ["", "", "", ""], "121": ["", "", "", ""], "122": ["", "", "", ""], "123": ["", "", "", ""], "124": ["", "", "", ""], "125": ["", "", "", ""], "126": ["", "", "", ""], "127": ["", "", "", ""], "128": ["", "", "", ""], "129": ["", "", "", ""], "130": ["", "", "", ""], "131": ["", "", "", ""], "132": ["", "", "", ""], "133": ["", "", "", ""], "134": ["", "", "", ""], "135": ["", "", "", ""], "136": ["", "", "", ""], "137": ["", "", "", ""], "138": ["", "", "", ""], "139": ["", "", "", ""], "140": ["", "", "", ""], "141": ["", "", "", ""], "142": ["", "", "", ""], "143": ["", "", "", ""], "144": ["", "", "", ""], "145": ["", "", "", ""], "146": ["", "", "", ""], "147": ["", "", "", ""], "148": ["", "", "", ""], "149": ["", "", "", ""], "150": ["", "", "", ""], "151": ["", "", "", ""], "152": ["", "", "", ""], "153": ["", "", "", ""], "154": ["", "", "", ""], "155": ["", "", "", ""], "156": ["", "", "", ""], "157": ["", "", "", ""], "158": ["", "", "", ""], "159": ["", "", "", ""], "160": ["", "", "", ""], "161": ["", "", "", ""], "162": ["", "", "", ""], "163": ["", "", "", ""], "164": ["", "", "", ""], "165": ["", "", "", ""], "166": ["", "", "", ""], "167": ["", "", "", ""], "168": ["", "", "", ""], "169": ["", "", "", ""], "170": ["", "", "", ""], "171": ["", "", "", ""], "172": ["", "", "", ""], "173": ["", "", "", ""], "174": ["", "", "", ""], "175": ["", "", "", ""], "176": ["", "", "", ""], "177": ["", "", "", ""], "178": ["", "", "", ""], "179": ["", "", "", ""], "180": ["", "", "", ""], "181": ["", "", "", ""], "182": ["", "", "", ""], "183": ["", "", "", ""], "184": ["", "", "", ""], "185": ["", "", "", ""], "186": ["", "", "", ""], "187": ["", "", "", ""], "188": ["", "", "", ""], "189": ["", "", "", ""], "190": ["", "", "", ""], "191": ["", "", "", ""], "192": ["", "", "", ""], "193": ["", "", "", ""], "194": ["", "", "", ""], "195": ["", "", "", ""], "196": ["", "", "", ""], "197": ["", "", "", ""], "198": ["", "", "", ""], "199": ["", "", "", ""], "200": ["", "", "", ""], "201": ["", "", "", ""], "202": ["", "", "", ""], "203": ["", "", "", ""], "204": ["", "", "", ""], "205": ["", "", "", ""], "206": ["", "", "", ""], "207": ["", "", "", ""], "208": ["", "", "", ""], "209": ["", "", "", ""], "210": ["", "", "", ""], "211": ["", "", "", ""], "212": ["", "", "", ""], "213": ["", "", "", ""], "214": ["", "", "", ""], "215": ["", "", "", ""], "216": ["", "", "", ""], "217": ["", "", "", ""], "218": ["", "", "", ""], "219": ["", "", "", ""], "220": ["", "", "", ""], "221": ["", "", "", ""], "222": ["", "", "", ""], "223": ["", "", "", ""], "224": ["", "", "", ""], "225": ["", "", "", ""], "226": ["", "", "", ""], "227": ["", "", "", ""], "228": ["", "", "", ""], "229": ["", "", "", ""], "230": ["", "", "", ""], "231": ["", "", "", ""], "232": ["", "", "", ""], "233": ["", "", "", ""], "234": ["", "", "", ""], "235": ["", "", "", ""], "236": ["", "", "", ""], "237": ["", "", "", ""], "238": ["", "", "", ""], "239": ["", "", "", ""], "240": ["", "", "", ""], "241": ["", "", "", ""], "242": ["", "", "", ""], "243": ["", "", "", ""], "244": ["", "", "", ""], "245": ["", "", "", ""], "246": ["", "", "", ""], "247": ["", "", "", ""], "248": ["", "", "", ""], "249": ["", "", "", ""], "250": ["", "", "", ""], "251": ["", "", "", ""], "252": ["", "", "", ""], "253": ["", "", "", ""], "254": ["", "", "", ""], "255": ["", "", "", ""], "256": ["", "", "", ""], "257": ["", "", "", ""], "258": ["", "", "", ""], "259": ["", "", "", ""], "260": ["", "", "", ""], "261": ["", "", "", ""], "262": ["", "", "", ""], "263": ["", "", "", ""], "264": ["", "", "", ""], "265": ["", "", "", ""], "266": ["", "", "", ""], "267": ["", "", "", ""], "268": ["", "", "", ""], "269": ["", "", "", ""], "270": ["", "", "", ""], "271": ["", "", "", ""], "272": ["", "", "", ""], "273": ["", "", "", ""], "274": ["", "", "", ""], "275": ["", "", "", ""], "276": ["", "", "", ""], "277": ["", "", "", ""], "278": ["", "", "", ""], "279": ["", "", "", ""], "280": ["", "", "", ""], "281": ["", "", "", ""], "282": ["", "", "", ""], "283": ["", "", "", ""], "284": ["", "", "", ""], "285": ["", "", "", ""], "286": ["", "", "", ""], "287": ["", "", "", ""], "288": ["", "", "", ""], "289": ["", "", "", ""], "290": ["", "", "", ""], "291": ["", "", "", ""], "292": ["", "", "", ""], "293": ["", "", "", ""], "294": ["", "", "", ""], "295": ["", "", "", ""], "296": ["", "", "", ""], "297": ["", "", "", ""], "298": ["", "", "", ""], "299": ["", "", "", ""], "300": ["", "", "", ""], "301": ["", "", "", ""], "302": ["", "", "", ""], "303": ["", "", "", ""], "304": ["", "", "", ""], "305": ["", "", "", ""], "306": ["", "", "", ""], "307": ["", "", "", ""], "308": ["", "", "", ""], "309": ["", "", "", ""], "310": ["", "", "", ""], "311": ["", "", "", ""], "312": ["", "", "", ""], "313": ["", "", "", ""], "314": ["", "", "", ""], "315": ["", "", "", ""], "316": ["", "", "", ""], "317": ["", "", "", ""], "318": ["", "", "", ""], "319": ["", "", "", ""], "320": ["", "", "", ""], "321": ["", "", "", ""], "322": ["", "", "", ""], "323": ["", "", "", ""], "324": ["", "", "", ""], "325": ["", "", "", ""], "326": ["", "", "", ""], "327": ["", "", "", ""], "328": ["", "", "", ""], "329": ["", "", "", ""], "330": ["", "", "", ""], "331": ["", "", "", ""], "332": ["", "", "", ""], "333": ["", "", "", ""], "334": ["", "", "", ""], "335": ["", "", "", ""], "336": ["", "", "", ""], "337": ["", "", "", ""], "338": ["", "", "", ""], "339": ["", "", "", ""], "340": ["", "", "", ""], "341": ["", "", "", ""], "342": ["", "", "", ""], "343": ["", "", "", ""], "344": ["", "", "", ""], "345": ["", "", "", ""], "346": ["", "", "", ""], "347": ["", "", "", ""], "348": ["", "", "", ""], "349": ["", "", "", ""], "350": ["", "", "", ""], "351": ["", "", "", ""], "352": ["", "", "", ""], "353": ["", "", "", ""], "354": ["", "", "", ""], "355": ["", "", "", ""], "356": ["", "", "", ""], "357": ["", "", "", ""], "358": ["", "", "", ""], "359": ["", "", "", ""], "360": ["", "", "", ""], "361": ["", "", "", ""], "362": ["", "", "", ""], "363": ["", "", "", ""], "364": ["", "", "", ""], "365": ["", "", "", ""], "366": ["", "", "", ""], "367": ["", "", "", ""], "368": ["", "", "", ""], "369": ["", "", "", ""], "370": ["", "", "", ""], "371": ["", "", "", ""], "372": ["", "", "", ""], "373": ["", "", "", ""], "374": ["", "", "", ""], "375": ["", "", "", ""], "376": ["", "", "", ""], "377": ["", "", "", ""], "378": ["", "", "", ""], "379": ["", "", "", ""], "380": ["", "", "", ""], "381": ["", "", "", ""], "382": ["", "", "", ""], "383": ["", "", "", ""], "384": ["", "", "", ""], "385": ["", "", "", ""], "386": ["", "", "", ""], "387": ["", "", "", ""], "388": ["", "", "", ""], "389": ["", "", "", ""], "390": ["", "", "", ""], "391": ["", "", "", ""], "392": ["", "", "", ""], "393": ["", "", "", ""], "394": ["", "", "", ""], "395": ["", "", "", ""], "396": ["", "", "", ""], "397": ["", "", "", ""], "398": ["", "", "", ""], "399": ["", "", "", ""], "400": ["", "", "", ""], "401": ["", "", "", ""], "402": ["", "", "", ""], "403": ["", "", "", ""], "404": ["", "", "", ""], "405": ["", "", "", ""], "406": ["", "", "", ""], "407": ["", "", "", ""], "408": ["", "", "", ""], "409": ["", "", "", ""], "410": ["", "", "", ""], "411": ["", "", "", ""], "412": ["", "", "", ""], "413": ["", "", "", ""], "414": ["", "", "", ""], "415": ["", "", "", ""], "416": ["", "", "", ""], "417": ["", "", "", ""], "418": ["", "", "", ""], "419": ["", "", "", ""], "420": ["", "", "", ""], "421": ["", "", "", ""], "422": ["", "", "", ""], "423": ["", "", "", ""], "424": ["", "", "", ""], "425": ["", "", "", ""], "426": ["", "", "", ""], "427": ["", "", "", ""], "428": ["", "", "", ""], "429": ["", "", "", ""], "430": ["", "", "", ""], "431": ["", "", "", ""], "432": ["", "", "", ""], "433": ["", "", "", ""], "434": ["", "", "", ""], "435": ["", "", "", ""], "436": ["", "", "", ""], "437": ["", "", "", ""], "438": ["", "", "", ""], "439": ["", "", "", ""], "440": ["", "", "", ""], "441": ["", "", "", ""], "442": ["", "", "", ""], "443": ["", "", "", ""], "444": ["", "", "", ""], "445": ["", "", "", ""], "446": ["", "", "", ""], "447": ["", "", "", ""], "448": ["", "", "", ""], "449": ["", "", "", ""], "450": ["", "", "", ""], "451": ["", "", "", ""], "452": ["", "", "", ""], "453": ["", "", "", ""], "454": ["", "", "", ""], "455": ["", "", "", ""], "456": ["", "", "", ""], "457": ["", "", "", ""], "458": ["", "", "", ""], "459": ["", "", "", ""], "460": ["", "", "", ""], "461": ["", "", "", ""], "462": ["", "", "", ""], "463": ["", "", "", ""], "464": ["", "", "", ""], "465": ["", "", "", ""], "466": ["", "", "", ""], "467": ["", "", "", ""], "468": ["", "", "", ""], "469": ["", "", "", ""], "470": ["", "", "", ""], "471": ["", "", "", ""], "472": ["", "", "", ""], "473": ["", "", "", ""], "474": ["", "", "", ""], "475": ["", "", "", ""], "476": ["", "", "", ""], "477": ["", "", "", ""], "478": ["", "", "", ""], "479": ["", "", "", ""], "480": ["", "", "", ""], "481": ["", "", "", ""], "482": ["", "", "", ""], "483": ["", "", "", ""], "484": ["", "", "", ""], "485": ["", "", "", ""], "486": ["", "", "", ""], "487": ["", "", "", ""], "488": ["", "", "", ""], "489": ["", "", "", ""], "490": ["", "", "", ""], "491": ["", "", "", ""], "492": ["", "", "", ""], "493": ["", "", "", ""], "494": ["", "", "", ""], "495": ["", "", "", ""], "496": ["", "", "", ""], "497": ["", "", "", ""], "498": ["", "", "", ""], "499": ["", "", "", ""], "500": ["", "", "", ""], "501": ["", "", "", ""], "502": ["", "", "", ""], "503": ["", "", "", ""], "504": ["", "", "", ""], "505": ["", "", "", ""], "506": ["", "", "", ""], "507": ["", "", "", ""], "508": ["", "", "", ""], "509": ["", "", "", ""], "510": ["", "", "", ""], "511": ["", "", "", ""], "512": ["", "", "", ""], "513": ["", "", "", ""], "514": ["", "", "", ""], "515": ["", "", "", ""], "516": ["", "", "", ""], "517": ["", "", "", ""], "518": ["", "", "", ""], "519": ["", "", "", ""], "520": ["", "", "", ""], "521": ["", "", "", ""], "522": ["", "", "", ""], "523": ["", "", "", ""], "524": ["", "", "", ""], "525": ["", "", "", ""], "526": ["", "", "", ""], "527": ["", "", "", ""], "528": ["", "", "", ""], "529": ["", "", "", ""], "530": ["", "", "", ""], "531": ["", "", "", ""], "532": ["", "", "", ""], "533": ["", "", "", ""], "534": ["", "", "", ""], "535": ["", "", "", ""], "536": ["", "", "", ""], "537": ["", "", "", ""], "538": ["", "", "", ""], "539": ["", "", "", ""], "540": ["", "", "", ""], "541": ["", "", "", ""], "542": ["", "", "", ""], "543": ["", "", "", ""], "544": ["", "", "", ""], "545": ["", "", "", ""], "546": ["", "", "", ""], "547": ["", "", "", ""], "548": ["", "", "", ""], "549": ["", "", "", ""], "550": ["", "", "", ""], "551": ["", "", "", ""], "552": ["", "", "", ""], "553": ["", "", "", ""], "554": ["", "", "", ""], "555": ["", "", "", ""], "556": ["", "", "", ""], "557": ["", "", "", ""], "558": ["", "", "", ""], "559": ["", "", "", ""], "560": ["", "", "", ""], "561": ["", "", "", ""], "562": ["", "", "", ""], "563": ["", "", "", ""], "564": ["", "", "", ""], "565": ["", "", "", ""], "566": ["", "", "", ""], "567": ["", "", "", ""], "568": ["", "", "", ""], "569": ["", "", "", ""], "570": ["", "", "", ""], "571": ["", "", "", ""], "572": ["", "", "", ""], "573": ["", "", "", ""], "574": ["", "", "", ""], "575": ["", "", "", ""], "576": ["", "", "", ""], "577": ["", "", "", ""], "578": ["", "", "", ""], "579": ["", "", "", ""], "580": ["", "", "", ""], "581": ["", "", "", ""], "582": ["", "", "", ""], "583": ["", "", "", ""], "584": ["", "", "", ""], "585": ["", "", "", ""], "586": ["", "", "", ""], "587": ["", "", "", ""], "588": ["", "", "", ""], "589": ["", "", "", ""], "590": ["", "", "", ""], "591": ["", "", "", ""], "592": ["", "", "", ""], "593": ["", "", "", ""], "594": ["", "", "", ""], "595": ["", "", "", ""], "596": ["", "", "", ""], "597": ["", "", "", ""], "598": ["", "", "", ""], "599": ["", "", "", ""], "600": ["", "", "", ""], "601": ["", "", "", ""], "602": ["", "", "", ""], "603": ["", "", "", ""], "604": ["", "", "", ""], "605": ["", "", "", ""], "606": ["", "", "", ""], "607": ["", "", "", ""], "608": ["", "", "", ""], "609": ["", "", "", ""], "610": ["", "", "", ""], "611": ["", "", "", ""], "612": ["", "", "", ""], "613": ["", "", "", ""], "614": ["", "", "", ""], "615": ["", "", "", ""], "616": ["", "", "", ""], "617": ["", "", "", ""], "618": ["", "", "", ""], "619": ["", "", "", ""], "620": ["", "", "", ""], "621": ["", "", "", ""], "622": ["", "", "", ""], "623": ["", "", "", ""], "624": ["", "", "", ""], "625": ["", "", "", ""], "626": ["", "", "", ""], "627": ["", "", "", ""], "628": ["", "", "", ""], "629": ["", "", "", ""], "630": ["", "", "", ""], "631": ["", "", "", ""], "632": ["", "", "", ""], "633": ["", "", "", ""], "634": ["", "", "", ""], "635": ["", "", "", ""], "636": ["", "", "", ""], "637": ["", "", "", ""], "638": ["", "", "", ""], "639": ["", "", "", ""], "640": ["", "", "", ""], "641": ["", "", "", ""], "642": ["", "", "", ""], "643": ["", "", "", ""], "644": ["", "", "", ""], "645": ["", "", "", ""], "646": ["", "", "", ""], "647": ["", "", "", ""], "648": ["", "", "", ""], "649": ["", "", "", ""], "650": ["", "", "", ""], "651": ["", "", "", ""], "652": ["", "", "", ""], "653": ["", "", "", ""], "654": ["", "", "", ""], "655": ["", "", "", ""], "656": ["", "", "", ""], "657": ["", "", "", ""], "658": ["", "", "", ""], "659": ["", "", "", ""], "660": ["", "", "", ""], "661": ["", "", "", ""], "662": ["", "", "", ""], "663": ["", "", "", ""], "664": ["", "", "", ""], "665": ["", "", "", ""], "666": ["", "", "", ""], "667": ["", "", "", ""], "668": ["", "", "", ""], "669": ["", "", "", ""], "670": ["", "", "", ""], "671": ["", "", "", ""], "672": ["", "", "", ""], "673": ["", "", "", ""], "674": ["", "", "", ""], "675": ["", "", "", ""], "676": ["", "", "", ""], "677": ["", "", "", ""], "678": ["", "", "", ""], "679": ["", "", "", ""], "680": ["", "", "", ""], "681": ["", "", "", ""], "682": ["", "", "", ""], "683": ["", "", "", ""], "684": ["", "", "", ""], "685": ["", "", "", ""], "686": ["", "", "", ""], "687": ["", "", "", ""], "688": ["", "", "", ""], "689": ["", "", "", ""], "690": ["", "", "", ""], "691": ["", "", "", ""], "692": ["", "", "", ""], "693": ["", "", "", ""], "694": ["", "", "", ""], "695": ["", "", "", ""], "696": ["", "", "", ""], "697": ["", "", "", ""], "698": ["", "", "", ""], "699": ["", "", "", ""], "700": ["", "", "", ""], "701": ["", "", "", ""], "702": ["", "", "", ""], "703": ["", "", "", ""], "704": ["", "", "", ""], "705": ["", "", "", ""], "706": ["", "", "", ""], "707": ["", "", "", ""], "708": ["", "", "", ""], "709": ["", "", "", ""], "710": ["", "", "", ""], "711": ["", "", "", ""], "712": ["", "", "", ""], "713": ["", "", "", ""], "714": ["", "", "", ""], "715": ["", "", "", ""], "716": ["", "", "", ""], "717": ["", "", "", ""], "718": ["", "", "", ""], "719": ["", "", "", ""], "720": ["", "", "", ""], "721": ["", "", "", ""], "722": ["", "", "", ""], "723": ["", "", "", ""], "724": ["", "", "", ""], "725": ["", "", "", ""], "726": ["", "", "", ""], "727": ["", "", "", ""], "728": ["", "", "", ""], "729": ["", "", "", ""], "730": ["", "", "", ""], "731": ["", "", "", ""], "732": ["", "", "", ""], "733": ["", "", "", ""], "734": ["", "", "", ""], "735": ["", "", "", ""], "736": ["", "", "", ""], "737": ["", "", "", ""], "738": ["", "", "", ""], "739": ["", "", "", ""], "740": ["", "", "", ""], "741": ["", "", "", ""], "742": ["", "", "", ""], "743": ["", "", "", ""], "744": ["", "", "", ""], "745": ["", "", "", ""], "746": ["", "", "", ""], "747": ["", "", "", ""], "748": ["", "", "", ""], "749": ["", "", "", ""], "750": ["", "", "", ""], "751": ["", "", "", ""], "752": ["", "", "", ""], "753": ["", "", "", ""], "754": ["", "", "", ""], "755": ["", "", "", ""], "756": ["", "", "", ""], "757": ["", "", "", ""], "758": ["", "", "", ""], "759": ["", "", "", ""], "760": ["", "", "", ""], "761": ["", "", "", ""], "762": ["", "", "", ""], "763": ["", "", "", ""], "764": ["", "", "", ""], "765": ["", "", "", ""], "766": ["", "", "", ""], "767": ["", "", "", ""], "768": ["", "", "", ""], "769": ["", "", "", ""], "770": ["", "", "", ""], "771": ["", "", "", ""], "772": ["", "", "", ""], "773": ["", "", "", ""], "774": ["", "", "", ""], "775": ["", "", "", ""], "776": ["", "", "", ""], "777": ["", "", "", ""], "778": ["", "", "", ""], "779": ["", "", "", ""], "780": ["", "", "", ""], "781": ["", "", "", ""], "782": ["", "", "", ""], "783": ["", "", "", ""], "784": ["", "", "", ""], "785": ["", "", "", ""], "786": ["", "", "", ""], "787": ["", "", "", ""], "788": ["", "", "", ""], "789": ["", "", "", ""], "790": ["", "", "", ""], "791": ["", "", "", ""], "792": ["", "", "", ""], "793": ["", "", "", ""], "794": ["", "", "", ""], "795": ["", "", "", ""], "796": ["", "", "", ""], "797": ["", "", "", ""], "798": ["", "", "", ""], "799": ["", "", "", ""], "800": ["", "", "", ""], "801": ["", "", "", ""], "802": ["", "", "", ""], "803": ["", "", "", ""], "804": ["", "", "", ""], "805": ["", "", "", ""], "806": ["", "", "", ""], "807": ["", "", "", ""], "808": ["", "", "", ""], "809": ["", "", "", ""], "810": ["", "", "", ""], "811": ["", "", "", ""], "812": ["", "", "", ""], "813": ["", "", "", ""], "814": ["", "", "", ""], "815": ["", "", "", ""], "816": ["", "", "", ""], "817": ["", "", "", ""], "818": ["", "", "", ""], "819": ["", "", "", ""], "820": ["", "", "", ""], "821": ["", "", "", ""], "822": ["", "", "", ""], "823": ["", "", "", ""], "824": ["", "", "", ""], "825": ["", "", "", ""], "826": ["", "", "", ""], "827": ["", "", "", ""], "828": ["", "", "", ""], "829": ["", "", "", ""], "830": ["", "", "", ""], "831": ["", "", "", ""], "832": ["", "", "", ""], "833": ["", "", "", ""], "834": ["", "", "", ""], "835": ["", "", "", ""], "836": ["", "", "", ""], "837": ["", "", "", ""], "838": ["", "", "", ""], "839": ["", "", "", ""], "840": ["", "", "", ""], "841": ["", "", "", ""], "842": ["", "", "", ""], "843": ["", "", "", ""], "844": ["", "", "", ""], "845": ["", "", "", ""], "846": ["", "", "", ""], "847": ["", "", "", ""], "848": ["", "", "", ""], "849": ["", "", "", ""], "850": ["", "", "", ""], "851": ["", "", "", ""], "852": ["", "", "", ""], "853": ["", "", "", ""], "854": ["", "", "", ""], "855": ["", "", "", ""], "856": ["", "", "", ""], "857": ["", "", "", ""], "858": ["", "", "", ""], "859": ["", "", "", ""], "860": ["", "", "", ""], "861": ["", "", "", ""], "862": ["", "", "", ""], "863": ["", "", "", ""], "864": ["", "", "", ""], "865": ["", "", "", ""], "866": ["", "", "", ""], "867": ["", "", "", ""], "868": ["", "", "", ""], "869": ["", "", "", ""], "870": ["", "", "", ""], "871": ["", "", "", ""], "872": ["", "", "", ""], "873": ["", "", "", ""], "874": ["", "", "", ""], "875": ["", "", "", ""], "876": ["", "", "", ""], "877": ["", "", "", ""], "878": ["", "", "", ""], "879": ["", "", "", ""], "880": ["", "", "", ""], "881": ["", "", "", ""], "882": ["", "", "", ""], "883": ["", "", "", ""], "884": ["", "", "", ""], "885": ["", "", "", ""], "886": ["", "", "", ""], "887": ["", "", "", ""], "888": ["", "", "", ""], "889": ["", "", "", ""], "890": ["", "", "", ""], "891": ["", "", "", ""], "892": ["", "", "", ""], "893": ["", "", "", ""], "894": ["", "", "", ""], "895": ["", "", "", ""], "896": ["", "", "", ""], "897": ["", "", "", ""], "898": ["", "", "", ""], "899": ["", "", "", ""], "900": ["", "", "", ""], "901": ["", "", "", ""], "902": ["", "", "", ""], "903": ["", "", "", ""], "904": ["", "", "", ""], "905": ["", "", "", ""], "906": ["", "", "", ""], "907": ["", "", "", ""], "908": ["", "", "", ""], "909": ["", "", "", ""], "910": ["", "", "", ""], "911": ["", "", "", ""], "912": ["", "", "", ""], "913": ["", "", "", ""], "914": ["", "", "", ""], "915": ["", "", "", ""], "916": ["", "", "", ""], "917": ["", "", "", ""], "918": ["", "", "", ""], "919": ["", "", "", ""], "920": ["", "", "", ""], "921": ["", "", "", ""], "922": ["", "", "", ""], "923": ["", "", "", ""], "924": ["", "", "", ""], "925": ["", "", "", ""], "926": ["", "", "", ""], "927": ["", "", "", ""], "928": ["", "", "", ""], "929": ["", "", "", ""], "930": ["", "", "", ""], "931": ["", "", "", ""], "932": ["", "", "", ""], "933": ["", "", "", ""], "934": ["", "", "", ""], "935": ["", "", "", ""], "936": ["", "", "", ""], "937": ["", "", "", ""], "938": ["", "", "", ""], "939": ["", "", "", ""], "940": ["", "", "", ""], "941": ["", "", "", ""], "942": ["", "", "", ""], "943": ["", "", "", ""], "944": ["", "", "", ""], "945": ["", "", "", ""], "946": ["", "", "", ""], "947": ["", "", "", ""], "948": ["", "", "", ""], "949": ["", "", "", ""], "950": ["", "", "", ""], "951": ["", "", "", ""], "952": ["", "", "", ""], "953": ["", "", "", ""], "954": ["", "", "", ""], "955": ["", "", "", ""], "956": ["", "", "", ""], "957": ["", "", "", ""], "958": ["", "", "", ""], "959": ["", "", "", ""], "960": ["", "", "", ""], "961": ["", "", "", ""], "962": ["", "", "", ""], "963": ["", "", "", ""], "964": ["", "", "", ""], "965": ["", "", "", ""], "966": ["", "", "", ""], "967": ["", "", "", ""], "968": ["", "", "", ""], "969": ["", "", "", ""], "970": ["", "", "", ""], "971": ["", "", "", ""], "972": ["", "", "", ""], "973": ["", "", "", ""], "974": ["", "", "", ""], "975": ["", "", "", ""], "976": ["", "", "", ""], "977": ["", "", "", ""], "978": ["", "", "", ""], "979": ["", "", "", ""], "980": ["", "", "", ""], "981": ["", "", "", ""], "982": ["", "", "", ""], "983": ["", "", "", ""], "984": ["", "", "", ""], "985": ["", "", "", ""], "986": ["", "", "", ""], "987": ["", "", "", ""], "988": ["", "", "", ""], "989": ["", "", "", ""], "990": ["", "", "", ""], "991": ["", "", "", ""], "992": ["", "", "", ""], "993": ["", "", "", ""], "994": ["", "", "", ""], "995": ["", "", "", ""], "996": ["", "", "", ""], "997": ["", "", "", ""], "998": ["", "", "", ""], "999": ["", "", "", ""], "1000": ["", "", "", ""], "1001": ["", "", "", ""], "1002": ["", "", "", ""], "1003": ["", "", "", ""], "1004": ["", "", "", ""], "1005": ["", "", "", ""], "1006": ["", "", "", ""], "1007": ["", "", "", ""], "1008": ["", "", "", ""], "1009": ["", "", "", ""], "1010": ["", "", "", ""], "1011": ["", "", "", ""], "1012": ["", "", "", ""], "1013": ["", "", "", ""], "1014": ["", "", "", ""], "1015": ["", "", "", ""], "1016": ["", "", "", ""], "1017": ["", "", "", ""], "1018": ["", "", "", ""], "1019": ["", "", "", ""], "1020": ["", "", "", ""], "1021": ["", "", "", ""], "1022": ["", "", "", ""], "1023": ["", "", "", ""], "1024": ["", "", "", ""], "1025": ["", "", "", ""], "1026": ["", "", "", ""], "1027": ["", "", "", ""], "1028": ["", "", "", ""], "1029": ["", "", "", ""], "1030": ["", "", "", ""], "1031": ["", "", "", ""], "1032": ["", "", "", ""], "1033": ["", "", "", ""], "1034": ["", "", "", ""], "1035": ["", "", "", ""], "1036": ["", "", "", ""], "1037": ["", "", "", ""], "1038": ["", "", "", ""], "1039": ["", "", "", ""], "1040": ["", "", "", ""], "1041": ["", "", "", ""], "1042": ["", "", "", ""], "1043": ["", "", "", ""], "1044": ["", "", "", ""], "1045": ["", "", "", ""], "1046": ["", "", "", ""], "1047": ["", "", "", ""], "1048": ["", "", "", ""], "1049": ["", "", "", ""], "1050": ["", "", "", ""], "1051": ["", "", "", ""], "1052": ["", "", "", ""], "1053": ["", "", "", ""], "1054": ["", "", "", ""], "1055": ["", "", "", ""], "1056": ["", "", "", ""], "1057": ["", "", "", ""], "1058": ["", "", "", ""], "1059": ["", "", "", ""], "1060": ["", "", "", ""], "1061": ["", "", "", ""], "1062": ["", "", "", ""], "1063": ["", "", "", ""], "1064": ["", "", "", ""], "1065": ["", "", "", ""], "1066": ["", "", "", ""], "1067": ["", "", "", ""], "1068": ["", "", "", ""], "1069": ["", "", "", ""], "1070": ["", "", "", ""], "1071": ["", "", "", ""], "1072": ["", "", "", ""], "1073": ["", "", "", ""], "1074": ["", "", "", ""], "1075": ["", "", "", ""], "1076": ["", "", "", ""], "1077": ["", "", "", ""], "1078": ["", "", "", ""], "1079": ["", "", "", ""], "1080": ["", "", "", ""], "1081": ["", "", "", ""], "1082": ["", "", "", ""], "1083": ["", "", "", ""], "1084": ["", "", "", ""], "1085": ["", "", "", ""], "1086": ["", "", "", ""], "1087": ["", "", "", ""], "1088": ["", "", "", ""], "1089": ["", "", "", ""], "1090": ["", "", "", ""], "1091": ["", "", "", ""], "1092": ["", "", "", ""], "1093": ["", "", "", ""], "1094": ["", "", "", ""], "1095": ["", "", "", ""], "1096": ["", "", "", ""], "1097": ["", "", "", ""], "1098": ["", "", "", ""], "1099": ["", "", "", ""], "1100": ["", "", "", ""], "1101": ["", "", "", ""], "1102": ["", "", "", ""], "1103": ["", "", "", ""], "1104": ["", "", "", ""], "1105": ["", "", "", ""], "1106": ["", "", "", ""], "1107": ["", "", "", ""], "1108": ["", "", "", ""], "1109": ["", "", "", ""], "1110": ["", "", "", ""], "1111": ["", "", "", ""], "1112": ["", "", "", ""], "1113": ["", "", "", ""], "1114": ["", "", "", ""], "1115": ["", "", "", ""], "1116": ["", "", "", ""], "1117": ["", "", "", ""], "1118": ["", "", "", ""], "1119": ["", "", "", ""], "1120": ["", "", "", ""], "1121": ["", "", "", ""], "1122": ["", "", "", ""], "1123": ["", "", "", ""], "1124": ["", "", "", ""], "1125": ["", "", "", ""], "1126": ["", "", "", ""], "1127": ["", "", "", ""], "1128": ["", "", "", ""], "1129": ["", "", "", ""], "1130": ["", "", "", ""], "1131": ["", "", "", ""], "1132": ["", "", "", ""], "1133": ["", "", "", ""], "1134": ["", "", "", ""], "1135": ["", "", "", ""], "1136": ["", "", "", ""], "1137": ["", "", "", ""], "1138": ["", "", "", ""], "1139": ["", "", "", ""], "1140": ["", "", "", ""], "1141": ["", "", "", ""], "1142": ["", "", "", ""], "1143": ["", "", "", ""], "1144": ["", "", "", ""], "1145": ["", "", "", ""], "1146": ["", "", "", ""], "1147": ["", "", "", ""], "1148": ["", "", "", ""], "1149": ["", "", "", ""], "1150": ["", "", "", ""], "1151": ["", "", "", ""], "1152": ["", "", "", ""], "1153": ["", "", "", ""], "1154": ["", "", "", ""], "1155": ["", "", "", ""], "1156": ["", "", "", ""], "1157": ["", "", "", ""], "1158": ["", "", "", ""], "1159": ["", "", "", ""], "1160": ["", "", "", ""], "1161": ["", "", "", ""], "1162": ["", "", "", ""], "1163": ["", "", "", ""], "1164": ["", "", "", ""], "1165": ["", "", "", ""], "1166": ["", "", "", ""], "1167": ["", "", "", ""], "1168": ["", "", "", ""], "1169": ["", "", "", ""], "1170": ["", "", "", ""], "1171": ["", "", "", ""], "1172": ["", "", "", ""], "1173": ["", "", "", ""], "1174": ["", "", "", ""], "1175": ["", "", "", ""], "1176": ["", "", "", ""], "1177": ["", "", "", ""], "1178": ["", "", "", ""], "1179": ["", "", "", ""], "1180": ["", "", "", ""], "1181": ["", "", "", ""], "1182": ["", "", "", ""], "1183": ["", "", "", ""], "1184": ["", "", "", ""], "1185": ["", "", "", ""], "1186": ["", "", "", ""], "1187": ["", "", "", ""], "1188": ["", "", "", ""], "1189": ["", "", "", ""], "1190": ["", "", "", ""], "1191": ["", "", "", ""], "1192": ["", "", "", ""], "1193": ["", "", "", ""], "1194": ["", "", "", ""], "1195": ["", "", "", ""], "1196": ["", "", "", ""], "1197": ["", "", "", ""], "1198": ["", "", "", ""], "1199": ["", "", "", ""], "1200": ["", "", "", ""], "1201": ["", "", "", ""], "1202": ["", "", "", ""], "1203": ["", "", "", ""], "1204": ["", "", "", ""], "1205": ["", "", "", ""], "1206": ["", "", "", ""], "1207": ["", "", "", ""], "1208": ["", "", "", ""], "1209": ["", "", "", ""], "1210": ["", "", "", ""], "1211": ["", "", "", ""], "1212": ["", "", "", ""], "1213": ["", "", "", ""], "1214": ["", "", "", ""], "1215": ["", "", "", ""], "1216": ["", "", "", ""], "1217": ["", "", "", ""], "1218": ["", "", "", ""], "1219": ["", "", "", ""], "1220": ["", "", "", ""], "1221": ["", "", "", ""], "1222": ["", "", "", ""], "1223": ["", "", "", ""], "1224": ["", "", "", ""], "1225": ["", "", "", ""], "1226": ["", "", "", ""], "1227": ["", "", "", ""], "1228": ["", "", "", ""], "1229": ["", "", "", ""], "1230": ["", "", "", ""], "1231": ["", "", "", ""], "1232": ["", "", "", ""], "1233": ["", "", "", ""], "1234": ["", "", "", ""], "1235": ["", "", "", ""], "1236": ["", "", "", ""], "1237": ["", "", "", ""], "1238": ["", "", "", ""], "1239": ["", "", "", ""], "1240": ["", "", "", ""], "1241": ["", "", "", ""], "1242": ["", "", "", ""], "1243": ["", "", "", ""], "1244": ["", "", "", ""], "1245": ["", "", "", ""], "1246": ["", "", "", ""], "1247": ["", "", "", ""], "1248": ["", "", "", ""], "1249": ["", "", "", ""], "1250": ["", "", "", ""], "1251": ["", "", "", ""], "1252": ["", "", "", ""], "1253": ["", "", "", ""], "1254": ["", "", "", ""], "1255": ["", "", "", ""], "1256": ["", "", "", ""], "1257": ["", "", "", ""], "1258": ["", "", "", ""], "1259": ["", "", "", ""], "1260": ["", "", "", ""], "1261": ["", "", "", ""], "1262": ["", "", "", ""], "1263": ["", "", "", ""], "1264": ["", "", "", ""], "1265": ["", "", "", ""], "1266": ["", "", "", ""], "1267": ["", "", "", ""], "1268": ["", "", "", ""], "1269": ["", "", "", ""], "1270": ["", "", "", ""], "1271": ["", "", "", ""], "1272": ["", "", "", ""], "1273": ["", "", "", ""], "1274": ["", "", "", ""], "1275": ["", "", "", ""], "1276": ["", "", "", ""], "1277": ["", "", "", ""], "1278": ["", "", "", ""], "1279": ["", "", "", ""], "1280": ["", "", "", ""], "1281": ["", "", "", ""], "1282": ["", "", "", ""], "1283": ["", "", "", ""], "1284": ["", "", "", ""], "1285": ["", "", "", ""], "1286": ["", "", "", ""], "1287": ["", "", "", ""], "1288": ["", "", "", ""], "1289": ["", "", "", ""], "1290": ["", "", "", ""], "1291": ["", "", "", ""], "1292": ["", "", "", ""], "1293": ["", "", "", ""], "1294": ["", "", "", ""], "1295": ["", "", "", ""], "1296": ["", "", "", ""], "1297": ["", "", "", ""], "1298": ["", "", "", ""], "1299": ["", "", "", ""], "1300": ["", "", "", ""], "1301": ["", "", "", ""], "1302": ["", "", "", ""], "1303": ["", "", "", ""], "1304": ["", "", "", ""], "1305": ["", "", "", ""], "1306": ["", "", "", ""], "1307": ["", "", "", ""], "1308": ["", "", "", ""], "1309": ["", "", "", ""], "1310": ["", "", "", ""], "1311": ["", "", "", ""], "1312": ["", "", "", ""], "1313": ["", "", "", ""], "1314": ["", "", "", ""], "1315": ["", "", "", ""], "1316": ["", "", "", ""], "1317": ["", "", "", ""], "1318": ["", "", "", ""], "1319": ["", "", "", ""], "1320": ["", "", "", ""], "1321": ["", "", "", ""], "1322": ["", "", "", ""], "1323": ["", "", "", ""], "1324": ["", "", "", ""], "1325": ["", "", "", ""], "1326": ["", "", "", ""], "1327": ["", "", "", ""], "1328": ["", "", "", ""], "1329": ["", "", "", ""], "1330": ["", "", "", ""], "1331": ["", "", "", ""], "1332": ["", "", "", ""], "1333": ["", "", "", ""], "1334": ["", "", "", ""], "1335": ["", "", "", ""], "1336": ["", "", "", ""], "1337": ["", "", "", ""], "1338": ["", "", "", ""], "1339": ["", "", "", ""], "1340": ["", "", "", ""], "1341": ["", "", "", ""], "1342": ["", "", "", ""], "1343": ["", "", "", ""], "1344": ["", "", "", ""], "1345": ["", "", "", ""], "1346": ["", "", "", ""], "1347": ["", "", "", ""], "1348": ["", "", "", ""], "1349": ["", "", "", ""], "1350": ["", "", "", ""], "1351": ["", "", "", ""], "1352": ["", "", "", ""], "1353": ["", "", "", ""], "1354": ["", "", "", ""], "1355": ["", "", "", ""], "1356": ["", "", "", ""], "1357": ["", "", "", ""], "1358": ["", "", "", ""], "1359": ["", "", "", ""], "1360": ["", "", "", ""], "1361": ["", "", "", ""], "1362": ["", "", "", ""], "1363": ["", "", "", ""], "1364": ["", "", "", ""], "1365": ["", "", "", ""], "1366": ["", "", "", ""], "1367": ["", "", "", ""], "1368": ["", "", "", ""], "1369": ["", "", "", ""], "1370": ["", "", "", ""], "1371": ["", "", "", ""], "1372": ["", "", "", ""], "1373": ["", "", "", ""], "1374": ["", "", "", ""], "1375": ["", "", "", ""], "1376": ["", "", "", ""], "1377": ["", "", "", ""], "1378": ["", "", "", ""], "1379": ["", "", "", ""], "1380": ["", "", "", ""], "1381": ["", "", "", ""], "1382": ["", "", "", ""], "1383": ["", "", "", ""], "1384": ["", "", "", ""], "1385": ["", "", "", ""], "1386": ["", "", "", ""], "1387": ["", "", "", ""], "1388": ["", "", "", ""], "1389": ["", "", "", ""], "1390": ["", "", "", ""], "1391": ["", "", "", ""], "1392": ["", "", "", ""], "1393": ["", "", "", ""], "1394": ["", "", "", ""], "1395": ["", "", "", ""], "1396": ["", "", "", ""], "1397": ["", "", "", ""], "1398": ["", "", "", ""], "1399": ["", "", "", ""], "1400": ["", "", "", ""], "1401": ["", "", "", ""], "1402": ["", "", "", ""], "1403": ["", "", "", ""], "1404": ["", "", "", ""], "1405": ["", "", "", ""], "1406": ["", "", "", ""], "1407": ["", "", "", ""], "1408": ["", "", "", ""], "1409": ["", "", "", ""], "1410": ["", "", "", ""], "1411": ["", "", "", ""], "1412": ["", "", "", ""], "1413": ["", "", "", ""], "1414": ["", "", "", ""], "1415": ["", "", "", ""], "1416": ["", "", "", ""], "1417": ["", "", "", ""], "1418": ["", "", "", ""], "1419": ["", "", "", ""], "1420": ["", "", "", ""], "1421": ["", "", "", ""], "1422": ["", "", "", ""], "1423": ["", "", "", ""], "1424": ["", "", "", ""], "1425": ["", "", "", ""], "1426": ["", "", "", ""], "1427": ["", "", "", ""], "1428": ["", "", "", ""], "1429": ["", "", "", ""], "1430": ["", "", "", ""], "1431": ["", "", "", ""], "1432": ["", "", "", ""], "1433": ["", "", "", ""], "1434": ["", "", "", ""], "1435": ["", "", "", ""], "1436": ["", "", "", ""], "1437": ["", "", "", ""], "1438": ["", "", "", ""], "1439": ["", "", "", ""], "1440": ["", "", "", ""], "1441": ["", "", "", ""], "1442": ["", "", "", ""], "1443": ["", "", "", ""], "1444": ["", "", "", ""], "1445": ["", "", "", ""], "1446": ["", "", "", ""], "1447": ["", "", "", ""], "1448": ["", "", "", ""], "1449": ["", "", "", ""], "1450": ["", "", "", ""], "1451": ["", "", "", ""], "1452": ["", "", "", ""], "1453": ["", "", "", ""], "1454": ["", "", "", ""], "1455": ["", "", "", ""], "1456": ["", "", "", ""], "1457": ["", "", "", ""], "1458": ["", "", "", ""], "1459": ["", "", "", ""], "1460": ["", "", "", ""], "1461": ["", "", "", ""], "1462": ["", "", "", ""], "1463": ["", "", "", ""], "1464": ["", "", "", ""], "1465": ["", "", "", ""], "1466": ["", "", "", ""], "1467": ["", "", "", ""], "1468": ["", "", "", ""], "1469": ["", "", "", ""], "1470": ["", "", "", ""], "1471": ["", "", "", ""], "1472": ["", "", "", ""], "1473": ["", "", "", ""], "1474": ["", "", "", ""], "1475": ["", "", "", ""], "1476": ["", "", "", ""], "1477": ["", "", "", ""], "1478": ["", "", "", ""], "1479": ["", "", "", ""], "1480": ["", "", "", ""], "1481": ["", "", "", ""], "1482": ["", "", "", ""], "1483": ["", "", "", ""], "1484": ["", "", "", ""], "1485": ["", "", "", ""], "1486": ["", "", "", ""], "1487": ["", "", "", ""], "1488": ["", "", "", ""], "1489": ["", "", "", ""], "1490": ["", "", "", ""], "1491": ["", "", "", ""], "1492": ["", "", "", ""], "1493": ["", "", "", ""], "1494": ["", "", "", ""], "1495": ["", "", "", ""], "1496": ["", "", "", ""], "1497": ["", "", "", ""], "1498": ["", "", "", ""], "1499": ["", "", "", ""], "1500": ["", "", "", ""], "1501": ["", "", "", ""], "1502": ["", "", "", ""], "1503": ["", "", "", ""], "1504": ["", "", "", ""], "1505": ["", "", "", ""], "1506": ["", "", "", ""], "1507": ["", "", "", ""], "1508": ["", "", "", ""], "1509": ["", "", "", ""], "1510": ["", "", "", ""], "1511": ["", "", "", ""], "1512": ["", "", "", ""], "1513": ["", "", "", ""], "1514": ["", "", "", ""], "1515": ["", "", "", ""], "1516": ["", "", "", ""], "1517": ["", "", "", ""], "1518": ["", "", "", ""], "1519": ["", "", "", ""], "1520": ["", "", "", ""], "1521": ["", "", "", ""], "1522": ["", "", "", ""], "1523": ["", "", "", ""], "1524": ["", "", "", ""], "1525": ["", "", "", ""], "1526": ["", "", "", ""], "1527": ["", "", "", ""], "1528": ["", "", "", ""], "1529": ["", "", "", ""], "1530": ["", "", "", ""], "1531": ["", "", "", ""], "1532": ["", "", "", ""], "1533": ["", "", "", ""], "1534": ["", "", "", ""], "1535": ["", "", "", ""], "1536": ["", "", "", ""], "1537": ["", "", "", ""], "1538": ["", "", "", ""], "1539": ["", "", "", ""], "1540": ["", "", "", ""], "1541": ["", "", "", ""], "1542": ["", "", "", ""], "1543": ["", "", "", ""], "1544": ["", "", "", ""], "1545": ["", "", "", ""], "1546": ["", "", "", ""], "1547": ["", "", "", ""], "1548": ["", "", "", ""], "1549": ["", "", "", ""], "1550": ["", "", "", ""], "1551": ["", "", "", ""], "1552": ["", "", "", ""], "1553": ["", "", "", ""], "1554": ["", "", "", ""], "1555": ["", "", "", ""], "1556": ["", "", "", ""], "1557": ["", "", "", ""], "1558": ["", "", "", ""], "1559": ["", "", "", ""], "1560": ["", "", "", ""], "1561": ["", "", "", ""], "1562": ["", "", "", ""], "1563": ["", "", "", ""], "1564": ["", "", "", ""], "1565": ["", "", "", ""], "1566": ["", "", "", ""], "1567": ["", "", "", ""], "1568": ["", "", "", ""], "1569": ["", "", "", ""], "1570": ["", "", "", ""], "1571": ["", "", "", ""], "1572": ["", "", "", ""], "1573": ["", "", "", ""], "1574": ["", "", "", ""], "1575": ["", "", "", ""], "1576": ["", "", "", ""], "1577": ["", "", "", ""], "1578": ["", "", "", ""], "1579": ["", "", "", ""], "1580": ["", "", "", ""], "1581": ["", "", "", ""], "1582": ["", "", "", ""], "1583": ["", "", "", ""], "1584": ["", "", "", ""], "1585": ["", "", "", ""], "1586": ["", "", "", ""], "1587": ["", "", "", ""], "1588": ["", "", "", ""], "1589": ["", "", "", ""], "1590": ["", "", "", ""], "1591": ["", "", "", ""], "1592": ["", "", "", ""], "1593": ["", "", "", ""], "1594": ["", "", "", ""], "1595": ["", "", "", ""], "1596": ["", "", "", ""], "1597": ["", "", "", ""], "1598": ["", "", "", ""], "1599": ["", "", "", ""], "1600": ["", "", "", ""], "1601": ["", "", "", ""], "1602": ["", "", "", ""], "1603": ["", "", "", ""], "1604": ["", "", "", ""], "1605": ["", "", "", ""], "1606": ["", "", "", ""], "1607": ["", "", "", ""], "1608": ["", "", "", ""], "1609": ["", "", "", ""], "1610": ["", "", "", ""], "1611": ["", "", "", ""], "1612": ["", "", "", ""], "1613": ["", "", "", ""], "1614": ["", "", "", ""], "1615": ["", "", "", ""], "1616": ["", "", "", ""], "1617": ["", "", "", ""], "1618": ["", "", "", ""], "1619": ["", "", "", ""], "1620": ["", "", "", ""], "1621": ["", "", "", ""], "1622": ["", "", "", ""], "1623": ["", "", "", ""], "1624": ["", "", "", ""], "1625": ["", "", "", ""], "1626": ["", "", "", ""], "1627": ["", "", "", ""], "1628": ["", "", "", ""], "1629": ["", "", "", ""], "1630": ["", "", "", ""], "1631": ["", "", "", ""], "1632": ["", "", "", ""], "1633": ["", "", "", ""], "1634": ["", "", "", ""], "1635": ["", "", "", ""], "1636": ["", "", "", ""], "1637": ["", "", "", ""], "1638": ["", "", "", ""], "1639": ["", "", "", ""], "1640": ["", "", "", ""], "1641": ["", "", "", ""], "1642": ["", "", "", ""], "1643": ["", "", "", ""], "1644": ["", "", "", ""], "1645": ["", "", "", ""], "1646": ["", "", "", ""], "1647": ["", "", "", ""], "1648": ["", "", "", ""], "1649": ["", "", "", ""], "1650": ["", "", "", ""], "1651": ["", "", "", ""], "1652": ["", "", "", ""], "1653": ["", "", "", ""], "1654": ["", "", "", ""], "1655": ["", "", "", ""], "1656": ["", "", "", ""], "1657": ["", "", "", ""], "1658": ["", "", "", ""], "1659": ["", "", "", ""], "1660": ["", "", "", ""], "1661": ["", "", "", ""], "1662": ["", "", "", ""], "1663": ["", "", "", ""], "1664": ["", "", "", ""], "1665": ["", "", "", ""], "1666": ["", "", "", ""], "1667": ["", "", "", ""], "1668": ["", "", "", ""], "1669": ["", "", "", ""], "1670": ["", "", "", ""], "1671": ["", "", "", ""], "1672": ["", "", "", ""], "1673": ["", "", "", ""], "1674": ["", "", "", ""], "1675": ["", "", "", ""], "1676": ["", "", "", ""], "1677": ["", "", "", ""], "1678": ["", "", "", ""], "1679": ["", "", "", ""], "1680": ["", "", "", ""], "1681": ["", "", "", ""], "1682": ["", "", "", ""], "1683": ["", "", "", ""], "1684": ["", "", "", ""], "1685": ["", "", "", ""], "1686": ["", "", "", ""], "1687": ["", "", "", ""], "1688": ["", "", "", ""], "1689": ["", "", "", ""], "1690": ["", "", "", ""], "1691": ["", "", "", ""], "1692": ["", "", "", ""], "1693": ["", "", "", ""], "1694": ["", "", "", ""], "1695": ["", "", "", ""], "1696": ["", "", "", ""], "1697": ["", "", "", ""], "1698": ["", "", "", ""], "1699": ["", "", "", ""], "1700": ["", "", "", ""], "1701": ["", "", "", ""], "1702": ["", "", "", ""], "1703": ["", "", "", ""], "1704": ["", "", "", ""], "1705": ["", "", "", ""], "1706": ["", "", "", ""], "1707": ["", "", "", ""], "1708": ["", "", "", ""], "1709": ["", "", "", ""], "1710": ["", "", "", ""], "1711": ["", "", "", ""], "1712": ["", "", "", ""], "1713": ["", "", "", ""], "1714": ["", "", "", ""], "1715": ["", "", "", ""], "1716": ["", "", "", ""], "1717": ["", "", "", ""], "1718": ["", "", "", ""], "1719": ["", "", "", ""], "1720": ["", "", "", ""], "1721": ["", "", "", ""], "1722": ["", "", "", ""], "1723": ["", "", "", ""], "1724": ["", "", "", ""], "1725": ["", "", "", ""], "1726": ["", "", "", ""], "1727": ["", "", "", ""], "1728": ["", "", "", ""], "1729": ["", "", "", ""], "1730": ["", "", "", ""], "1731": ["", "", "", ""], "1732": ["", "", "", ""], "1733": ["", "", "", ""], "1734": ["", "", "", ""], "1735": ["", "", "", ""], "1736": ["", "", "", ""], "1737": ["", "", "", ""], "1738": ["", "", "", ""], "1739": ["", "", "", ""], "1740": ["", "", "", ""], "1741": ["", "", "", ""], "1742": ["", "", "", ""], "1743": ["", "", "", ""], "1744": ["", "", "", ""], "1745": ["", "", "", ""], "1746": ["", "", "", ""], "1747": ["", "", "", ""], "1748": ["", "", "", ""], "1749": ["", "", "", ""], "1750": ["", "", "", ""], "1751": ["", "", "", ""], "1752": ["", "", "", ""], "1753": ["", "", "", ""], "1754": ["", "", "", ""], "1755": ["", "", "", ""], "1756": ["", "", "", ""], "1757": ["", "", "", ""], "1758": ["", "", "", ""], "1759": ["", "", "", ""], "1760": ["", "", "", ""], "1761": ["", "", "", ""], "1762": ["", "", "", ""], "1763": ["", "", "", ""], "1764": ["", "", "", ""], "1765": ["", "", "", ""], "1766": ["", "", "", ""], "1767": ["", "", "", ""], "1768": ["", "", "", ""], "1769": ["", "", "", ""], "1770": ["", "", "", ""], "1771": ["", "", "", ""], "1772": ["", "", "", ""], "1773": ["", "", "", ""], "1774": ["", "", "", ""], "1775": ["", "", "", ""], "1776": ["", "", "", ""], "1777": ["", "", "", ""], "1778": ["", "", "", ""], "1779": ["", "", "", ""], "1780": ["", "", "", ""], "1781": ["", "", "", ""], "1782": ["", "", "", ""], "1783": ["", "", "", ""], "1784": ["", "", "", ""], "1785": ["", "", "", ""], "1786": ["", "", "", ""], "1787": ["", "", "", ""], "1788": ["", "", "", ""], "1789": ["", "", "", ""], "1790": ["", "", "", ""], "1791": ["", "", "", ""], "1792": ["", "", "", ""], "1793": ["", "", "", ""], "1794": ["", "", "", ""], "1795": ["", "", "", ""], "1796": ["", "", "", ""], "1797": ["", "", "", ""], "1798": ["", "", "", ""], "1799": ["", "", "", ""], "1800": ["", "", "", ""], "1801": ["", "", "", ""], "1802": ["", "", "", ""], "1803": ["", "", "", ""], "1804": ["", "", "", ""], "1805": ["", "", "", ""], "1806": ["", "", "", ""], "1807": ["", "", "", ""], "1808": ["", "", "", ""], "1809": ["", "", "", ""], "1810": ["", "", "", ""], "1811": ["", "", "", ""], "1812": ["", "", "", ""], "1813": ["", "", "", ""], "1814": ["", "", "", ""], "1815": ["", "", "", ""], "1816": ["", "", "", ""], "1817": ["", "", "", ""], "1818": ["", "", "", ""], "1819": ["", "", "", ""], "1820": ["", "", "", ""], "1821": ["", "", "", ""], "1822": ["", "", "", ""], "1823": ["", "", "", ""], "1824": ["", "", "", ""], "1825": ["", "", "", ""], "1826": ["", "", "", ""], "1827": ["", "", "", ""], "1828": ["", "", "", ""], "1829": ["", "", "", ""], "1830": ["", "", "", ""], "1831": ["", "", "", ""], "1832": ["", "", "", ""], "1833": ["", "", "", ""], "1834": ["", "", "", ""], "1835": ["", "", "", ""], "1836": ["", "", "", ""], "1837": ["", "", "", ""], "1838": ["", "", "", ""], "1839": ["", "", "", ""], "1840": ["", "", "", ""], "1841": ["", "", "", ""], "1842": ["", "", "", ""], "1843": ["", "", "", ""], "1844": ["", "", "", ""], "1845": ["", "", "", ""], "1846": ["", "", "", ""], "1847": ["", "", "", ""], "1848": ["", "", "", ""], "1849": ["", "", "", ""], "1850": ["", "", "", ""], "1851": ["", "", "", ""], "1852": ["", "", "", ""], "1853": ["", "", "", ""], "1854": ["", "", "", ""], "1855": ["", "", "", ""], "1856": ["", "", "", ""], "1857": ["", "", "", ""], "1858": ["", "", "", ""], "1859": ["", "", "", ""], "1860": ["", "", "", ""], "1861": ["", "", "", ""], "1862": ["", "", "", ""], "1863": ["", "", "", ""], "1864": ["", "", "", ""], "1865": ["", "", "", ""], "1866": ["", "", "", ""], "1867": ["", "", "", ""], "1868": ["", "", "", ""], "1869": ["", "", "", ""], "1870": ["", "", "", ""], "1871": ["", "", "", ""], "1872": ["", "", "", ""], "1873": ["", "", "", ""], "1874": ["", "", "", ""], "1875": ["", "", "", ""], "1876": ["", "", "", ""], "1877": ["", "", "", ""], "1878": ["", "", "", ""], "1879": ["", "", "", ""], "1880": ["", "", "", ""], "1881": ["", "", "", ""], "1882": ["", "", "", ""], "1883": ["", "", "", ""], "1884": ["", "", "", ""], "1885": ["", "", "", ""], "1886": ["", "", "", ""], "1887": ["", "", "", ""], "1888": ["", "", "", ""], "1889": ["", "", "", ""], "1890": ["", "", "", ""], "1891": ["", "", "", ""], "1892": ["", "", "", ""], "1893": ["", "", "", ""], "1894": ["", "", "", ""], "1895": ["", "", "", ""], "1896": ["", "", "", ""], "1897": ["", "", "", ""], "1898": ["", "", "", ""], "1899": ["", "", "", ""], "1900": ["", "", "", ""], "1901": ["", "", "", ""], "1902": ["", "", "", ""], "1903": ["", "", "", ""], "1904": ["", "", "", ""], "1905": ["", "", "", ""], "1906": ["", "", "", ""], "1907": ["", "", "", ""], "1908": ["", "", "", ""], "1909": ["", "", "", ""], "1910": ["", "", "", ""], "1911": ["", "", "", ""], "1912": ["", "", "", ""], "1913": ["", "", "", ""], "1914": ["", "", "", ""], "1915": ["", "", "", ""], "1916": ["", "", "", ""], "1917": ["", "", "", ""], "1918": ["", "", "", ""], "1919": ["", "", "", ""], "1920": ["", "", "", ""], "1921": ["", "", "", ""], "1922": ["", "", "", ""], "1923": ["", "", "", ""], "1924": ["", "", "", ""], "1925": ["", "", "", ""], "1926": ["", "", "", ""], "1927": ["", "", "", ""], "1928": ["", "", "", ""], "1929": ["", "", "", ""], "1930": ["", "", "", ""], "1931": ["", "", "", ""], "1932": ["", "", "", ""], "1933": ["", "", "", ""], "1934": ["", "", "", ""], "1935": ["", "", "", ""], "1936": ["", "", "", ""], "1937": ["", "", "", ""], "1938": ["", "", "", ""], "1939": ["", "", "", ""], "1940": ["", "", "", ""], "1941": ["", "", "", ""], "1942": ["", "", "", ""], "1943": ["", "", "", ""], "1944": ["", "", "", ""], "1945": ["", "", "", ""], "1946": ["", "", "", ""], "1947": ["", "", "", ""], "1948": ["", "", "", ""], "1949": ["", "", "", ""], "1950": ["", "", "", ""], "1951": ["", "", "", ""], "1952": ["", "", "", ""], "1953": ["", "", "", ""], "1954": ["", "", "", ""], "1955": ["", "", "", ""], "1956": ["", "", "", ""], "1957": ["", "", "", ""], "1958": ["", "", "", ""], "1959": ["", "", "", ""], "1960": ["", "", "", ""], "1961": ["", "", "", ""], "1962": ["", "", "", ""], "1963": ["", "", "", ""], "1964": ["", "", "", ""], "1965": ["", "", "", ""], "1966": ["", "", "", ""], "1967": ["", "", "", ""], "1968": ["", "", "", ""], "1969": ["", "", "", ""], "1970": ["", "", "", ""], "1971": ["", "", "", ""], "1972": ["", "", "", ""], "1973": ["", "", "", ""], "1974": ["", "", "", ""], "1975": ["", "", "", ""], "1976": ["", "", "", ""], "1977": ["", "", "", ""], "1978": ["", "", "", ""], "1979": ["", "", "", ""], "1980": ["", "", "", ""], "1981": ["", "", "", ""], "1982": ["", "", "", ""], "1983": ["", "", "", ""], "1984": ["", "", "", ""], "1985": ["", "", "", ""], "1986": ["", "", "", ""], "1987": ["", "", "", ""], "1988": ["", "", "", ""], "1989": ["", "", "", ""], "1990": ["", "", "", ""], "1991": ["", "", "", ""], "1992": ["", "", "", ""], "1993": ["", "", "", ""], "1994": ["", "", "", ""], "1995": ["", "", "", ""], "1996": ["", "", "", ""], "1997": ["", "", "", ""], "1998": ["", "", "", ""], "1999": ["", "", "", ""], "2000": ["", "", "", ""], "2001": ["", "", "", ""], "2002": ["", "", "", ""], "2003": ["", "", "", ""], "2004": ["", "", "", ""], "2005": ["", "", "", ""], "2006": ["", "", "", ""], "2007": ["", "", "", ""], "2008": ["", "", "", ""], "2009": ["", "", "", ""], "2010": ["", "", "", ""], "2011": ["", "", "", ""], "2012": ["", "", "", ""], "2013": ["", "", "", ""], "2014": ["", "", "", ""], "2015": ["", "", "", ""], "2016": ["", "", "", ""], "2017": ["", "", "", ""], "2018": ["", "", "", ""], "2019": ["", "", "", ""], "2020": ["", "", "", ""], "2021": ["", "", "", ""], "2022": ["", "", "", ""], "2023": ["", "", "", ""], "2024": ["", "", "", ""], "2025": ["", "", "", ""], "2026": ["", "", "", ""], "2027": ["", "", "", ""], "2028": ["", "", "", ""], "2029": ["", "", "", ""], "2030": ["", "", "", ""], "2031": ["", "", "", ""], "2032": ["", "", "", ""], "2033": ["", "", "", ""], "2034": ["", "", "", ""], "2035": ["", "", "", ""], "2036": ["", "", "", ""], "2037": ["", "", "", ""], "2038": ["", "", "", ""], "2039": ["", "", "", ""], "2040": ["", "", "", ""], "2041": ["", "", "", ""], "2042": ["", "", "", ""], "2043": ["", "", "", ""], "2044": ["", "", "", ""], "2045": ["", "", "", ""], "2046": ["", "", "", ""], "2047": ["", "", "", ""], "2048": ["", "", "", ""], "2049": ["", "", "", ""], "2050": ["", "", "", ""], "2051": ["", "", "", ""], "2052": ["", "", "", ""], "2053": ["", "", "", ""], "2054": ["", "", "", ""], "2055": ["", "", "", ""], "2056": ["", "", "", ""], "2057": ["", "", "", ""], "2058": ["", "", "", ""], "2059": ["", "", "", ""], "2060": ["", "", "", ""], "2061": ["", "", "", ""], "2062": ["", "", "", ""], "2063": ["", "", "", ""], "2064": ["", "", "", ""], "2065": ["", "", "", ""], "2066": ["", "", "", ""], "2067": ["", "", "", ""], "2068": ["", "", "", ""], "2069": ["", "", "", ""], "2070": ["", "", "", ""], "2071": ["", "", "", ""], "2072": ["", "", "", ""], "2073": ["", "", "", ""], "2074": ["", "", "", ""], "2075": ["", "", "", ""], "2076": ["", "", "", ""], "2077": ["", "", "", ""], "2078": ["", "", "", ""], "2079": ["", "", "", ""], "2080": ["", "", "", ""], "2081": ["", "", "", ""], "2082": ["", "", "", ""], "2083": ["", "", "", ""], "2084": ["", "", "", ""], "2085": ["", "", "", ""], "2086": ["", "", "", ""], "2087": ["", "", "", ""], "2088": ["", "", "", ""], "2089": ["", "", "", ""], "2090": ["", "", "", ""], "2091": ["", "", "", ""], "2092": ["", "", "", ""], "2093": ["", "", "", ""], "2094": ["", "", "", ""], "2095": ["", "", "", ""], "2096": ["", "", "", ""], "2097": ["", "", "", ""], "2098": ["", "", "", ""], "2099": ["", "", "", ""], "2100": ["", "", "", ""], "2101": ["", "", "", ""], "2102": ["", "", "", ""], "2103": ["", "", "", ""], "2104": ["", "", "", ""], "2105": ["", "", "", ""], "2106": ["", "", "", ""], "2107": ["", "", "", ""], "2108": ["", "", "", ""], "2109": ["", "", "", ""], "2110": ["", "", "", ""], "2111": ["", "", "", ""], "2112": ["", "", "", ""], "2113": ["", "", "", ""], "2114": ["", "", "", ""], "2115": ["", "", "", ""], "2116": ["", "", "", ""], "2117": ["", "", "", ""], "2118": ["", "", "", ""], "2119": ["", "", "", ""], "2120": ["", "", "", ""], "2121": ["", "", "", ""], "2122": ["", "", "", ""], "2123": ["", "", "", ""], "2124": ["", "", "", ""], "2125": ["", "", "", ""], "2126": ["", "", "", ""], "2127": ["", "", "", ""], "2128": ["", "", "", ""], "2129": ["", "", "", ""], "2130": ["", "", "", ""], "2131": ["", "", "", ""], "2132": ["", "", "", ""], "2133": ["", "", "", ""], "2134": ["", "", "", ""], "2135": ["", "", "", ""], "2136": ["", "", "", ""], "2137": ["", "", "", ""], "2138": ["", "", "", ""], "2139": ["", "", "", ""], "2140": ["", "", "", ""], "2141": ["", "", "", ""], "2142": ["", "", "", ""], "2143": ["", "", "", ""], "2144": ["", "", "", ""], "2145": ["", "", "", ""], "2146": ["", "", "", ""], "2147": ["", "", "", ""], "2148": ["", "", "", ""], "2149": ["", "", "", ""], "2150": ["", "", "", ""], "2151": ["", "", "", ""], "2152": ["", "", "", ""], "2153": ["", "", "", ""], "2154": ["", "", "", ""], "2155": ["", "", "", ""], "2156": ["", "", "", ""], "2157": ["", "", "", ""], "2158": ["", "", "", ""], "2159": ["", "", "", ""], "2160": ["", "", "", ""], "2161": ["", "", "", ""], "2162": ["", "", "", ""], "2163": ["", "", "", ""], "2164": ["", "", "", ""], "2165": ["", "", "", ""], "2166": ["", "", "", ""], "2167": ["", "", "", ""], "2168": ["", "", "", ""], "2169": ["", "", "", ""], "2170": ["", "", "", ""], "2171": ["", "", "", ""], "2172": ["", "", "", ""], "2173": ["", "", "", ""], "2174": ["", "", "", ""], "2175": ["", "", "", ""], "2176": ["", "", "", ""], "2177": ["", "", "", ""], "2178": ["", "", "", ""], "2179": ["", "", "", ""], "2180": ["", "", "", ""], "2181": ["", "", "", ""], "2182": ["", "", "", ""], "2183": ["", "", "", ""], "2184": ["", "", "", ""], "2185": ["", "", "", ""], "2186": ["", "", "", ""], "2187": ["", "", "", ""], "2188": ["", "", "", ""], "2189": ["", "", "", ""], "2190": ["", "", "", ""], "2191": ["", "", "", ""], "2192": ["", "", "", ""], "2193": ["", "", "", ""], "2194": ["", "", "", ""], "2195": ["", "", "", ""], "2196": ["", "", "", ""], "2197": ["", "", "", ""], "2198": ["", "", "", ""], "2199": ["", "", "", ""], "2200": ["", "", "", ""], "2201": ["", "", "", ""], "2202": ["", "", "", ""], "2203": ["", "", "", ""], "2204": ["", "", "", ""], "2205": ["", "", "", ""], "2206": ["", "", "", ""], "2207": ["", "", "", ""], "2208": ["", "", "", ""], "2209": ["", "", "", ""], "2210": ["", "", "", ""], "2211": ["", "", "", ""], "2212": ["", "", "", ""], "2213": ["", "", "", ""], "2214": ["", "", "", ""], "2215": ["", "", "", ""], "2216": ["", "", "", ""], "2217": ["", "", "", ""], "2218": ["", "", "", ""], "2219": ["", "", "", ""], "2220": ["", "", "", ""], "2221": ["", "", "", ""], "2222": ["", "", "", ""], "2223": ["", "", "", ""], "2224": ["", "", "", ""], "2225": ["", "", "", ""], "2226": ["", "", "", ""], "2227": ["", "", "", ""], "2228": ["", "", "", ""], "2229": ["", "", "", ""], "2230": ["", "", "", ""], "2231": ["", "", "", ""], "2232": ["", "", "", ""], "2233": ["", "", "", ""], "2234": ["", "", "", ""], "2235": ["", "", "", ""], "2236": ["", "", "", ""], "2237": ["", "", "", ""], "2238": ["", "", "", ""], "2239": ["", "", "", ""], "2240": ["", "", "", ""], "2241": ["", "", "", ""], "2242": ["", "", "", ""], "2243": ["", "", "", ""], "2244": ["", "", "", ""], "2245": ["", "", "", ""], "2246": ["", "", "", ""], "2247": ["", "", "", ""], "2248": ["", "", "", ""], "2249": ["", "", "", ""], "2250": ["", "", "", ""], "2251": ["", "", "", ""], "2252": ["", "", "", ""], "2253": ["", "", "", ""], "2254": ["", "", "", ""], "2255": ["", "", "", ""], "2256": ["", "", "", ""], "2257": ["", "", "", ""], "2258": ["", "", "", ""], "2259": ["", "", "", ""], "2260": ["", "", "", ""], "2261": ["", "", "", ""], "2262": ["", "", "", ""], "2263": ["", "", "", ""], "2264": ["", "", "", ""], "2265": ["", "", "", ""], "2266": ["", "", "", ""], "2267": ["", "", "", ""], "2268": ["", "", "", ""], "2269": ["", "", "", ""], "2270": ["", "", "", ""], "2271": ["", "", "", ""], "2272": ["", "", "", ""], "2273": ["", "", "", ""], "2274": ["", "", "", ""], "2275": ["", "", "", ""], "2276": ["", "", "", ""], "2277": ["", "", "", ""], "2278": ["", "", "", ""], "2279": ["", "", "", ""], "2280": ["", "", "", ""], "2281": ["", "", "", ""], "2282": ["", "", "", ""], "2283": ["", "", "", ""], "2284": ["", "", "", ""], "2285": ["", "", "", ""], "2286": ["", "", "", ""], "2287": ["", "", "", ""], "2288": ["", "", "", ""], "2289": ["", "", "", ""], "2290": ["", "", "", ""], "2291": ["", "", "", ""], "2292": ["", "", "", ""], "2293": ["", "", "", ""], "2294": ["", "", "", ""], "2295": ["", "", "", ""], "2296": ["", "", "", ""], "2297": ["", "", "", ""], "2298": ["", "", "", ""], "2299": ["", "", "", ""], "2300": ["", "", "", ""], "2301": ["", "", "", ""], "2302": ["", "", "", ""], "2303": ["", "", "", ""], "2304": ["", "", "", ""], "2305": ["", "", "", ""], "2306": ["", "", "", ""], "2307": ["", "", "", ""], "2308": ["", "", "", ""], "2309": ["", "", "", ""], "2310": ["", "", "", ""], "2311": ["", "", "", ""], "2312": ["", "", "", ""], "2313": ["", "", "", ""], "2314": ["", "", "", ""], "2315": ["", "", "", ""], "2316": ["", "", "", ""], "2317": ["", "", "", ""], "2318": ["", "", "", ""], "2319": ["", "", "", ""], "2320": ["", "", "", ""], "2321": ["", "", "", ""], "2322": ["", "", "", ""], "2323": ["", "", "", ""], "2324": ["", "", "", ""], "2325": ["", "", "", ""], "2326": ["", "", "", ""], "2327": ["", "", "", ""], "2328": ["", "", "", ""], "2329": ["", "", "", ""], "2330": ["", "", "", ""], "2331": ["", "", "", ""], "2332": ["", "", "", ""], "2333": ["", "", "", ""], "2334": ["", "", "", ""], "2335": ["", "", "", ""], "2336": ["", "", "", ""], "2337": ["", "", "", ""], "2338": ["", "", "", ""], "2339": ["", "", "", ""], "2340": ["", "", "", ""], "2341": ["", "", "", ""], "2342": ["", "", "", ""], "2343": ["", "", "", ""], "2344": ["", "", "", ""], "2345": ["", "", "", ""], "2346": ["", "", "", ""], "2347": ["", "", "", ""], "2348": ["", "", "", ""], "2349": ["", "", "", ""], "2350": ["", "", "", ""], "2351": ["", "", "", ""], "2352": ["", "", "", ""], "2353": ["", "", "", ""], "2354": ["", "", "", ""], "2355": ["", "", "", ""], "2356": ["", "", "", ""], "2357": ["", "", "", ""], "2358": ["", "", "", ""], "2359": ["", "", "", ""], "2360": ["", "", "", ""], "2361": ["", "", "", ""], "2362": ["", "", "", ""], "2363": ["", "", "", ""], "2364": ["", "", "", ""], "2365": ["", "", "", ""], "2366": ["", "", "", ""], "2367": ["", "", "", ""], "2368": ["", "", "", ""], "2369": ["", "", "", ""], "2370": ["", "", "", ""], "2371": ["", "", "", ""], "2372": ["", "", "", ""], "2373": ["", "", "", ""], "2374": ["", "", "", ""], "2375": ["", "", "", ""], "2376": ["", "", "", ""], "2377": ["", "", "", ""], "2378": ["", "", "", ""], "2379": ["", "", "", ""], "2380": ["", "", "", ""], "2381": ["", "", "", ""], "2382": ["", "", "", ""], "2383": ["", "", "", ""], "2384": ["", "", "", ""], "2385": ["", "", "", ""], "2386": ["", "", "", ""], "2387": ["", "", "", ""], "2388": ["", "", "", ""], "2389": ["", "", "", ""], "2390": ["", "", "", ""], "2391": ["", "", "", ""], "2392": ["", "", "", ""], "2393": ["", "", "", ""], "2394": ["", "", "", ""], "2395": ["", "", "", ""], "2396": ["", "", "", ""], "2397": ["", "", "", ""], "2398": ["", "", "", ""], "2399": ["", "", "", ""], "2400": ["", "", "", ""], "2401": ["", "", "", ""], "2402": ["", "", "", ""], "2403": ["", "", "", ""], "2404": ["", "", "", ""], "2405": ["", "", "", ""], "2406": ["", "", "", ""], "2407": ["", "", "", ""], "2408": ["", "", "", ""], "2409": ["", "", "", ""], "2410": ["", "", "", ""], "2411": ["", "", "", ""], "2412": ["", "", "", ""], "2413": ["", "", "", ""], "2414": ["", "", "", ""], "2415": ["", "", "", ""], "2416": ["", "", "", ""], "2417": ["", "", "", ""], "2418": ["", "", "", ""], "2419": ["", "", "", ""], "2420": ["", "", "", ""], "2421": ["", "", "", ""], "2422": ["", "", "", ""], "2423": ["", "", "", ""], "2424": ["", "", "", ""], "2425": ["", "", "", ""], "2426": ["", "", "", ""], "2427": ["", "", "", ""], "2428": ["", "", "", ""], "2429": ["", "", "", ""], "2430": ["", "", "", ""], "2431": ["", "", "", ""], "2432": ["", "", "", ""], "2433": ["", "", "", ""], "2434": ["", "", "", ""], "2435": ["", "", "", ""], "2436": ["", "", "", ""], "2437": ["", "", "", ""], "2438": ["", "", "", ""], "2439": ["", "", "", ""], "2440": ["", "", "", ""], "2441": ["", "", "", ""], "2442": ["", "", "", ""], "2443": ["", "", "", ""], "2444": ["", "", "", ""], "2445": ["", "", "", ""], "2446": ["", "", "", ""], "2447": ["", "", "", ""], "2448": ["", "", "", ""], "2449": ["", "", "", ""], "2450": ["", "", "", ""], "2451": ["", "", "", ""], "2452": ["", "", "", ""], "2453": ["", "", "", ""], "2454": ["", "", "", ""], "2455": ["", "", "", ""], "2456": ["", "", "", ""], "2457": ["", "", "", ""], "2458": ["", "", "", ""], "2459": ["", "", "", ""], "2460": ["", "", "", ""], "2461": ["", "", "", ""], "2462": ["", "", "", ""], "2463": ["", "", "", ""], "2464": ["", "", "", ""], "2465": ["", "", "", ""], "2466": ["", "", "", ""], "2467": ["", "", "", ""], "2468": ["", "", "", ""], "2469": ["", "", "", ""], "2470": ["", "", "", ""], "2471": ["", "", "", ""], "2472": ["", "", "", ""], "2473": ["", "", "", ""], "2474": ["", "", "", ""], "2475": ["", "", "", ""], "2476": ["", "", "", ""], "2477": ["", "", "", ""], "2478": ["", "", "", ""], "2479": ["", "", "", ""], "2480": ["", "", "", ""], "2481": ["", "", "", ""], "2482": ["", "", "", ""], "2483": ["", "", "", ""], "2484": ["", "", "", ""], "2485": ["", "", "", ""], "2486": ["", "", "", ""], "2487": ["", "", "", ""], "2488": ["", "", "", ""], "2489": ["", "", "", ""], "2490": ["", "", "", ""], "2491": ["", "", "", ""], "2492": ["", "", "", ""], "2493": ["", "", "", ""], "2494": ["", "", "", ""], "2495": ["", "", "", ""], "2496": ["", "", "", ""], "2497": ["", "", "", ""], "2498": ["", "", "", ""], "2499": ["", "", "", ""], "2500": ["", "", "", ""], "2501": ["", "", "", ""], "2502": ["", "", "", ""], "2503": ["", "", "", ""], "2504": ["", "", "", ""], "2505": ["", "", "", ""], "2506": ["", "", "", ""], "2507": ["", "", "", ""], "2508": ["", "", "", ""], "2509": ["", "", "", ""], "2510": ["", "", "", ""], "2511": ["", "", "", ""], "2512": ["", "", "", ""], "2513": ["", "", "", ""], "2514": ["", "", "", ""], "2515": ["", "", "", ""], "2516": ["", "", "", ""], "2517": ["", "", "", ""], "2518": ["", "", "", ""], "2519": ["", "", "", ""], "2520": ["", "", "", ""], "2521": ["", "", "", ""], "2522": ["", "", "", ""], "2523": ["", "", "", ""], "2524": ["", "", "", ""], "2525": ["", "", "", ""], "2526": ["", "", "", ""], "2527": ["", "", "", ""], "2528": ["", "", "", ""], "2529": ["", "", "", ""], "2530": ["", "", "", ""], "2531": ["", "", "", ""], "2532": ["", "", "", ""], "2533": ["", "", "", ""], "2534": ["", "", "", ""], "2535": ["", "", "", ""], "2536": ["", "", "", ""], "2537": ["", "", "", ""], "2538": ["", "", "", ""], "2539": ["", "", "", ""], "2540": ["", "", "", ""], "2541": ["", "", "", ""], "2542": ["", "", "", ""], "2543": ["", "", "", ""], "2544": ["", "", "", ""], "2545": ["", "", "", ""], "2546": ["", "", "", ""], "2547": ["", "", "", ""], "2548": ["", "", "", ""], "2549": ["", "", "", ""], "2550": ["", "", "", ""], "2551": ["", "", "", ""], "2552": ["", "", "", ""], "2553": ["", "", "", ""], "2554": ["", "", "", ""], "2555": ["", "", "", ""], "2556": ["", "", "", ""], "2557": ["", "", "", ""], "2558": ["", "", "", ""], "2559": ["", "", "", ""], "2560": ["", "", "", ""], "2561": ["", "", "", ""], "2562": ["", "", "", ""], "2563": ["", "", "", ""], "2564": ["", "", "", ""], "2565": ["", "", "", ""], "2566": ["", "", "", ""], "2567": ["", "", "", ""], "2568": ["", "", "", ""], "2569": ["", "", "", ""], "2570": ["", "", "", ""], "2571": ["", "", "", ""], "2572": ["", "", "", ""], "2573": ["", "", "", ""], "2574": ["", "", "", ""], "2575": ["", "", "", ""], "2576": ["", "", "", ""], "2577": ["", "", "", ""], "2578": ["", "", "", ""], "2579": ["", "", "", ""], "2580": ["", "", "", ""], "2581": ["", "", "", ""], "2582": ["", "", "", ""], "2583": ["", "", "", ""], "2584": ["", "", "", ""], "2585": ["", "", "", ""], "2586": ["", "", "", ""], "2587": ["", "", "", ""], "2588": ["", "", "", ""], "2589": ["", "", "", ""], "2590": ["", "", "", ""], "2591": ["", "", "", ""], "2592": ["", "", "", ""], "2593": ["", "", "", ""], "2594": ["", "", "", ""], "2595": ["", "", "", ""], "2596": ["", "", "", ""], "2597": ["", "", "", ""], "2598": ["", "", "", ""], "2599": ["", "", "", ""], "2600": ["", "", "", ""], "2601": ["", "", "", ""], "2602": ["", "", "", ""], "2603": ["", "", "", ""], "2604": ["", "", "", ""], "2605": ["", "", "", ""], "2606": ["", "", "", ""], "2607": ["", "", "", ""], "2608": ["", "", "", ""], "2609": ["", "", "", ""], "2610": ["", "", "", ""], "2611": ["", "", "", ""], "2612": ["", "", "", ""], "2613": ["", "", "", ""], "2614": ["", "", "", ""], "2615": ["", "", "", ""], "2616": ["", "", "", ""], "2617": ["", "", "", ""], "2618": ["", "", "", ""], "2619": ["", "", "", ""], "2620": ["", "", "", ""], "2621": ["", "", "", ""], "2622": ["", "", "", ""], "2623": ["", "", "", ""], "2624": ["", "", "", ""], "2625": ["", "", "", ""], "2626": ["", "", "", ""], "2627": ["", "", "", ""], "2628": ["", "", "", ""], "2629": ["", "", "", ""], "2630": ["", "", "", ""], "2631": ["", "", "", ""], "2632": ["", "", "", ""], "2633": ["", "", "", ""], "2634": ["", "", "", ""], "2635": ["", "", "", ""], "2636": ["", "", "", ""], "2637": ["", "", "", ""], "2638": ["", "", "", ""], "2639": ["", "", "", ""], "2640": ["", "", "", ""], "2641": ["", "", "", ""], "2642": ["", "", "", ""], "2643": ["", "", "", ""], "2644": ["", "", "", ""], "2645": ["", "", "", ""], "2646": ["", "", "", ""], "2647": ["", "", "", ""], "2648": ["", "", "", ""], "2649": ["", "", "", ""], "2650": ["", "", "", ""], "2651": ["", "", "", ""], "2652": ["", "", "", ""], "2653": ["", "", "", ""], "2654": ["", "", "", ""], "2655": ["", "", "", ""], "2656": ["", "", "", ""], "2657": ["", "", "", ""], "2658": ["", "", "", ""], "2659": ["", "", "", ""], "2660": ["", "", "", ""], "2661": ["", "", "", ""], "2662": ["", "", "", ""], "2663": ["", "", "", ""], "2664": ["", "", "", ""], "2665": ["", "", "", ""], "2666": ["", "", "", ""], "2667": ["", "", "", ""], "2668": ["", "", "", ""], "2669": ["", "", "", ""], "2670": ["", "", "", ""], "2671": ["", "", "", ""], "2672": ["", "", "", ""], "2673": ["", "", "", ""], "2674": ["", "", "", ""], "2675": ["", "", "", ""], "2676": ["", "", "", ""], "2677": ["", "", "", ""], "2678": ["", "", "", ""], "2679": ["", "", "", ""], "2680": ["", "", "", ""], "2681": ["", "", "", ""], "2682": ["", "", "", ""], "2683": ["", "", "", ""], "2684": ["", "", "", ""], "2685": ["", "", "", ""], "2686": ["", "", "", ""], "2687": ["", "", "", ""], "2688": ["", "", "", ""], "2689": ["", "", "", ""], "2690": ["", "", "", ""], "2691": ["", "", "", ""], "2692": ["", "", "", ""], "2693": ["", "", "", ""], "2694": ["", "", "", ""], "2695": ["", "", "", ""], "2696": ["", "", "", ""], "2697": ["", "", "", ""], "2698": ["", "", "", ""], "2699": ["", "", "", ""], "2700": ["", "", "", ""], "2701": ["", "", "", ""], "2702": ["", "", "", ""], "2703": ["", "", "", ""], "2704": ["", "", "", ""], "2705": ["", "", "", ""], "2706": ["", "", "", ""], "2707": ["", "", "", ""], "2708": ["", "", "", ""], "2709": ["", "", "", ""], "2710": ["", "", "", ""], "2711": ["", "", "", ""], "2712": ["", "", "", ""], "2713": ["", "", "", ""], "2714": ["", "", "", ""], "2715": ["", "", "", ""], "2716": ["", "", "", ""], "2717": ["", "", "", ""], "2718": ["", "", "", ""], "2719": ["", "", "", ""], "2720": ["", "", "", ""], "2721": ["", "", "", ""], "2722": ["", "", "", ""], "2723": ["", "", "", ""], "2724": ["", "", "", ""], "2725": ["", "", "", ""], "2726": ["", "", "", ""], "2727": ["", "", "", ""], "2728": ["", "", "", ""], "2729": ["", "", "", ""], "2730": ["", "", "", ""], "2731": ["", "", "", ""], "2732": ["", "", "", ""], "2733": ["", "", "", ""], "2734": ["", "", "", ""], "2735": ["", "", "", ""], "2736": ["", "", "", ""], "2737": ["", "", "", ""], "2738": ["", "", "", ""], "2739": ["", "", "", ""], "2740": ["", "", "", ""], "2741": ["", "", "", ""], "2742": ["", "", "", ""], "2743": ["", "", "", ""], "2744": ["", "", "", ""], "2745": ["", "", "", ""], "2746": ["", "", "", ""], "2747": ["", "", "", ""], "2748": ["", "", "", ""], "2749": ["", "", "", ""], "2750": ["", "", "", ""], "2751": ["", "", "", ""], "2752": ["", "", "", ""], "2753": ["", "", "", ""], "2754": ["", "", "", ""], "2755": ["", "", "", ""], "2756": ["", "", "", ""], "2757": ["", "", "", ""], "2758": ["", "", "", ""], "2759": ["", "", "", ""], "2760": ["", "", "", ""], "2761": ["", "", "", ""], "2762": ["", "", "", ""], "2763": ["", "", "", ""], "2764": ["", "", "", ""], "2765": ["", "", "", ""], "2766": ["", "", "", ""], "2767": ["", "", "", ""], "2768": ["", "", "", ""], "2769": ["", "", "", ""], "2770": ["", "", "", ""], "2771": ["", "", "", ""], "2772": ["", "", "", ""], "2773": ["", "", "", ""], "2774": ["", "", "", ""], "2775": ["", "", "", ""], "2776": ["", "", "", ""], "2777": ["", "", "", ""], "2778": ["", "", "", ""], "2779": ["", "", "", ""], "2780": ["", "", "", ""], "2781": ["", "", "", ""], "2782": ["", "", "", ""], "2783": ["", "", "", ""], "2784": ["", "", "", ""], "2785": ["", "", "", ""], "2786": ["", "", "", ""], "2787": ["", "", "", ""], "2788": ["", "", "", ""], "2789": ["", "", "", ""], "2790": ["", "", "", ""], "2791": ["", "", "", ""], "2792": ["", "", "", ""], "2793": ["", "", "", ""], "2794": ["", "", "", ""], "2795": ["", "", "", ""], "2796": ["", "", "", ""], "2797": ["", "", "", ""], "2798": ["", "", "", ""], "2799": ["", "", "", ""], "2800": ["", "", "", ""], "2801": ["", "", "", ""], "2802": ["", "", "", ""], "2803": ["", "", "", ""], "2804": ["", "", "", ""], "2805": ["", "", "", ""], "2806": ["", "", "", ""], "2807": ["", "", "", ""], "2808": ["", "", "", ""], "2809": ["", "", "", ""], "2810": ["", "", "", ""], "2811": ["", "", "", ""], "2812": ["", "", "", ""], "2813": ["", "", "", ""], "2814": ["", "", "", ""], "2815": ["", "", "", ""], "2816": ["", "", "", ""], "2817": ["", "", "", ""], "2818": ["", "", "", ""], "2819": ["", "", "", ""], "2820": ["", "", "", ""], "2821": ["", "", "", ""], "2822": ["", "", "", ""], "2823": ["", "", "", ""], "2824": ["", "", "", ""], "2825": ["", "", "", ""], "2826": ["", "", "", ""], "2827": ["", "", "", ""], "2828": ["", "", "", ""], "2829": ["", "", "", ""], "2830": ["", "", "", ""], "2831": ["", "", "", ""], "2832": ["", "", "", ""], "2833": ["", "", "", ""], "2834": ["", "", "", ""], "2835": ["", "", "", ""], "2836": ["", "", "", ""], "2837": ["", "", "", ""], "2838": ["", "", "", ""], "2839": ["", "", "", ""], "2840": ["", "", "", ""], "2841": ["", "", "", ""], "2842": ["", "", "", ""], "2843": ["", "", "", ""], "2844": ["", "", "", ""], "2845": ["", "", "", ""], "2846": ["", "", "", ""], "2847": ["", "", "", ""], "2848": ["", "", "", ""], "2849": ["", "", "", ""], "2850": ["", "", "", ""], "2851": ["", "", "", ""], "2852": ["", "", "", ""], "2853": ["", "", "", ""], "2854": ["", "", "", ""], "2855": ["", "", "", ""], "2856": ["", "", "", ""], "2857": ["", "", "", ""], "2858": ["", "", "", ""], "2859": ["", "", "", ""], "2860": ["", "", "", ""], "2861": ["", "", "", ""], "2862": ["", "", "", ""], "2863": ["", "", "", ""], "2864": ["", "", "", ""], "2865": ["", "", "", ""], "2866": ["", "", "", ""], "2867": ["", "", "", ""], "2868": ["", "", "", ""], "2869": ["", "", "", ""], "2870": ["", "", "", ""], "2871": ["", "", "", ""], "2872": ["", "", "", ""], "2873": ["", "", "", ""], "2874": ["", "", "", ""], "2875": ["", "", "", ""], "2876": ["", "", "", ""], "2877": ["", "", "", ""], "2878": ["", "", "", ""], "2879": ["", "", "", ""], "2880": ["", "", "", ""], "2881": ["", "", "", ""], "2882": ["", "", "", ""], "2883": ["", "", "", ""], "2884": ["", "", "", ""], "2885": ["", "", "", ""], "2886": ["", "", "", ""], "2887": ["", "", "", ""], "2888": ["", "", "", ""], "2889": ["", "", "", ""], "2890": ["", "", "", ""], "2891": ["", "", "", ""], "2892": ["", "", "", ""], "2893": ["", "", "", ""], "2894": ["", "", "", ""], "2895": ["", "", "", ""], "2896": ["", "", "", ""], "2897": ["", "", "", ""], "2898": ["", "", "", ""], "2899": ["", "", "", ""], "2900": ["", "", "", ""], "2901": ["", "", "", ""], "2902": ["", "", "", ""], "2903": ["", "", "", ""], "2904": ["", "", "", ""], "2905": ["", "", "", ""], "2906": ["", "", "", ""], "2907": ["", "", "", ""], "2908": ["", "", "", ""], "2909": ["", "", "", ""], "2910": ["", "", "", ""], "2911": ["", "", "", ""], "2912": ["", "", "", ""], "2913": ["", "", "", ""], "2914": ["", "", "", ""], "2915": ["", "", "", ""], "2916": ["", "", "", ""], "2917": ["", "", "", ""], "2918": ["", "", "", ""], "2919": ["", "", "", ""], "2920": ["", "", "", ""], "2921": ["", "", "", ""], "2922": ["", "", "", ""], "2923": ["", "", "", ""], "2924": ["", "", "", ""], "2925": ["", "", "", ""], "2926": ["", "", "", ""], "2927": ["", "", "", ""], "2928": ["", "", "", ""], "2929": ["", "", "", ""], "2930": ["", "", "", ""], "2931": ["", "", "", ""], "2932": ["", "", "", ""], "2933": ["", "", "", ""], "2934": ["", "", "", ""], "2935": ["", "", "", ""], "2936": ["", "", "", ""], "2937": ["", "", "", ""], "2938": ["", "", "", ""], "2939": ["", "", "", ""], "2940": ["", "", "", ""], "2941": ["", "", "", ""], "2942": ["", "", "", ""], "2943": ["", "", "", ""], "2944": ["", "", "", ""], "2945": ["", "", "", ""], "2946": ["", "", "", ""], "2947": ["", "", "", ""], "2948": ["", "", "", ""], "2949": ["", "", "", ""], "2950": ["", "", "", ""], "2951": ["", "", "", ""], "2952": ["", "", "", ""], "2953": ["", "", "", ""], "2954": ["", "", "", ""], "2955": ["", "", "", ""], "2956": ["", "", "", ""], "2957": ["", "", "", ""], "2958": ["", "", "", ""], "2959": ["", "", "", ""], "2960": ["", "", "", ""], "2961": ["", "", "", ""], "2962": ["", "", "", ""], "2963": ["", "", "", ""], "2964": ["", "", "", ""], "2965": ["", "", "", ""], "2966": ["", "", "", ""], "2967": ["", "", "", ""], "2968": ["", "", "", ""], "2969": ["", "", "", ""], "2970": ["", "", "", ""], "2971": ["", "", "", ""], "2972": ["", "", "", ""], "2973": ["", "", "", ""], "2974": ["", "", "", ""], "2975": ["", "", "", ""], "2976": ["", "", "", ""], "2977": ["", "", "", ""], "2978": ["", "", "", ""], "2979": ["", "", "", ""], "2980": ["", "", "", ""], "2981": ["", "", "", ""], "2982": ["", "", "", ""], "2983": ["", "", "", ""], "2984": ["", "", "", ""], "2985": ["", "", "", ""], "2986": ["", "", "", ""], "2987": ["", "", "", ""], "2988": ["", "", "", ""], "2989": ["", "", "", ""], "2990": ["", "", "", ""], "2991": ["", "", "", ""], "2992": ["", "", "", ""], "2993": ["", "", "", ""], "2994": ["", "", "", ""], "2995": ["", "", "", ""], "2996": ["", "", "", ""], "2997": ["", "", "", ""], "2998": ["", "", "", ""], "2999": ["", "", "", ""], "3000": ["", "", "", ""], "3001": ["", "", "", ""], "3002": ["", "", "", ""], "3003": ["", "", "", ""], "3004": ["", "", "", ""], "3005": ["", "", "", ""], "3006": ["", "", "", ""], "3007": ["", "", "", ""], "3008": ["", "", "", ""], "3009": ["", "", "", ""], "3010": ["", "", "", ""], "3011": ["", "", "", ""], "3012": ["", "", "", ""], "3013": ["", "", "", ""], "3014": ["", "", "", ""], "3015": ["", "", "", ""], "3016": ["", "", "", ""], "3017": ["", "", "", ""], "3018": ["", "", "", ""], "3019": ["", "", "", ""], "3020": ["", "", "", ""], "3021": ["", "", "", ""], "3022": ["", "", "", ""], "3023": ["", "", "", ""], "3024": ["", "", "", ""], "3025": ["", "", "", ""], "3026": ["", "", "", ""], "3027": ["", "", "", ""], "3028": ["", "", "", ""], "3029": ["", "", "", ""], "3030": ["", "", "", ""], "3031": ["", "", "", ""], "3032": ["", "", "", ""], "3033": ["", "", "", ""], "3034": ["", "", "", ""], "3035": ["", "", "", ""], "3036": ["", "", "", ""], "3037": ["", "", "", ""], "3038": ["", "", "", ""], "3039": ["", "", "", ""], "3040": ["", "", "", ""], "3041": ["", "", "", ""], "3042": ["", "", "", ""], "3043": ["", "", "", ""], "3044": ["", "", "", ""], "3045": ["", "", "", ""], "3046": ["", "", "", ""], "3047": ["", "", "", ""], "3048": ["", "", "", ""], "3049": ["", "", "", ""], "3050": ["", "", "", ""], "3051": ["", "", "", ""], "3052": ["", "", "", ""], "3053": ["", "", "", ""], "3054": ["", "", "", ""], "3055": ["", "", "", ""], "3056": ["", "", "", ""], "3057": ["", "", "", ""], "3058": ["", "", "", ""], "3059": ["", "", "", ""], "3060": ["", "", "", ""], "3061": ["", "", "", ""], "3062": ["", "", "", ""], "3063": ["", "", "", ""], "3064": ["", "", "", ""], "3065": ["", "", "", ""], "3066": ["", "", "", ""], "3067": ["", "", "", ""], "3068": ["", "", "", ""], "3069": ["", "", "", ""], "3070": ["", "", "", ""], "3071": ["", "", "", ""], "3072": ["", "", "", ""], "3073": ["", "", "", ""], "3074": ["", "", "", ""], "3075": ["", "", "", ""], "3076": ["", "", "", ""], "3077": ["", "", "", ""], "3078": ["", "", "", ""], "3079": ["", "", "", ""], "3080": ["", "", "", ""], "3081": ["", "", "", ""], "3082": ["", "", "", ""], "3083": ["", "", "", ""], "3084": ["", "", "", ""], "3085": ["", "", "", ""], "3086": ["", "", "", ""], "3087": ["", "", "", ""], "3088": ["", "", "", ""], "3089": ["", "", "", ""], "3090": ["", "", "", ""], "3091": ["", "", "", ""], "3092": ["", "", "", ""], "3093": ["", "", "", ""], "3094": ["", "", "", ""], "3095": ["", "", "", ""], "3096": ["", "", "", ""], "3097": ["", "", "", ""], "3098": ["", "", "", ""], "3099": ["", "", "", ""], "3100": ["", "", "", ""], "3101": ["", "", "", ""], "3102": ["", "", "", ""], "3103": ["", "", "", ""], "3104": ["", "", "", ""], "3105": ["", "", "", ""], "3106": ["", "", "", ""], "3107": ["", "", "", ""], "3108": ["", "", "", ""], "3109": ["", "", "", ""], "3110": ["", "", "", ""], "3111": ["", "", "", ""], "3112": ["", "", "", ""], "3113": ["", "", "", ""], "3114": ["", "", "", ""], "3115": ["", "", "", ""], "3116": ["", "", "", ""], "3117": ["", "", "", ""], "3118": ["", "", "", ""], "3119": ["", "", "", ""], "3120": ["", "", "", ""], "3121": ["", "", "", ""], "3122": ["", "", "", ""], "3123": ["", "", "", ""], "3124": ["", "", "", ""], "3125": ["", "", "", ""], "3126": ["", "", "", ""], "3127": ["", "", "", ""], "3128": ["", "", "", ""], "3129": ["", "", "", ""], "3130": ["", "", "", ""], "3131": ["", "", "", ""], "3132": ["", "", "", ""], "3133": ["", "", "", ""], "3134": ["", "", "", ""], "3135": ["", "", "", ""], "3136": ["", "", "", ""], "3137": ["", "", "", ""], "3138": ["", "", "", ""], "3139": ["", "", "", ""], "3140": ["", "", "", ""], "3141": ["", "", "", ""], "3142": ["", "", "", ""], "3143": ["", "", "", ""], "3144": ["", "", "", ""], "3145": ["", "", "", ""], "3146": ["", "", "", ""], "3147": ["", "", "", ""], "3148": ["", "", "", ""], "3149": ["", "", "", ""], "3150": ["", "", "", ""], "3151": ["", "", "", ""], "3152": ["", "", "", ""], "3153": ["", "", "", ""], "3154": ["", "", "", ""], "3155": ["", "", "", ""], "3156": ["", "", "", ""], "3157": ["", "", "", ""], "3158": ["", "", "", ""], "3159": ["", "", "", ""], "3160": ["", "", "", ""], "3161": ["", "", "", ""], "3162": ["", "", "", ""], "3163": ["", "", "", ""], "3164": ["", "", "", ""], "3165": ["", "", "", ""], "3166": ["", "", "", ""], "3167": ["", "", "", ""], "3168": ["", "", "", ""], "3169": ["", "", "", ""], "3170": ["", "", "", ""], "3171": ["", "", "", ""], "3172": ["", "", "", ""], "3173": ["", "", "", ""], "3174": ["", "", "", ""], "3175": ["", "", "", ""], "3176": ["", "", "", ""], "3177": ["", "", "", ""], "3178": ["", "", "", ""], "3179": ["", "", "", ""], "3180": ["", "", "", ""], "3181": ["", "", "", ""], "3182": ["", "", "", ""], "3183": ["", "", "", ""], "3184": ["", "", "", ""], "3185": ["", "", "", ""], "3186": ["", "", "", ""], "3187": ["", "", "", ""], "3188": ["", "", "", ""], "3189": ["", "", "", ""], "3190": ["", "", "", ""], "3191": ["", "", "", ""], "3192": ["", "", "", ""], "3193": ["", "", "", ""], "3194": ["", "", "", ""], "3195": ["", "", "", ""], "3196": ["", "", "", ""], "3197": ["", "", "", ""], "3198": ["", "", "", ""], "3199": ["", "", "", ""], "3200": ["", "", "", ""], "3201": ["", "", "", ""], "3202": ["", "", "", ""], "3203": ["", "", "", ""], "3204": ["", "", "", ""], "3205": ["", "", "", ""], "3206": ["", "", "", ""], "3207": ["", "", "", ""], "3208": ["", "", "", ""], "3209": ["", "", "", ""], "3210": ["", "", "", ""], "3211": ["", "", "", ""], "3212": ["", "", "", ""], "3213": ["", "", "", ""], "3214": ["", "", "", ""], "3215": ["", "", "", ""], "3216": ["", "", "", ""], "3217": ["", "", "", ""], "3218": ["", "", "", ""], "3219": ["", "", "", ""], "3220": ["", "", "", ""], "3221": ["", "", "", ""], "3222": ["", "", "", ""], "3223": ["", "", "", ""], "3224": ["", "", "", ""], "3225": ["", "", "", ""], "3226": ["", "", "", ""], "3227": ["", "", "", ""], "3228": ["", "", "", ""], "3229": ["", "", "", ""], "3230": ["", "", "", ""], "3231": ["", "", "", ""], "3232": ["", "", "", ""], "3233": ["", "", "", ""], "3234": ["", "", "", ""], "3235": ["", "", "", ""], "3236": ["", "", "", ""], "3237": ["", "", "", ""], "3238": ["", "", "", ""], "3239": ["", "", "", ""], "3240": ["", "", "", ""], "3241": ["", "", "", ""], "3242": ["", "", "", ""], "3243": ["", "", "", ""], "3244": ["", "", "", ""], "3245": ["", "", "", ""], "3246": ["", "", "", ""], "3247": ["", "", "", ""], "3248": ["", "", "", ""], "3249": ["", "", "", ""], "3250": ["", "", "", ""], "3251": ["", "", "", ""], "3252": ["", "", "", ""], "3253": ["", "", "", ""], "3254": ["", "", "", ""], "3255": ["", "", "", ""], "3256": ["", "", "", ""], "3257": ["", "", "", ""], "3258": ["", "", "", ""], "3259": ["", "", "", ""], "3260": ["", "", "", ""], "3261": ["", "", "", ""], "3262": ["", "", "", ""], "3263": ["", "", "", ""], "3264": ["", "", "", ""], "3265": ["", "", "", ""], "3266": ["", "", "", ""], "3267": ["", "", "", ""], "3268": ["", "", "", ""], "3269": ["", "", "", ""], "3270": ["", "", "", ""], "3271": ["", "", "", ""], "3272": ["", "", "", ""], "3273": ["", "", "", ""], "3274": ["", "", "", ""], "3275": ["", "", "", ""], "3276": ["", "", "", ""], "3277": ["", "", "", ""], "3278": ["", "", "", ""], "3279": ["", "", "", ""], "3280": ["", "", "", ""], "3281": ["", "", "", ""], "3282": ["", "", "", ""], "3283": ["", "", "", ""], "3284": ["", "", "", ""], "3285": ["", "", "", ""], "3286": ["", "", "", ""], "3287": ["", "", "", ""], "3288": ["", "", "", ""], "3289": ["", "", "", ""], "3290": ["", "", "", ""], "3291": ["", "", "", ""], "3292": ["", "", "", ""], "3293": ["", "", "", ""], "3294": ["", "", "", ""], "3295": ["", "", "", ""], "3296": ["", "", "", ""], "3297": ["", "", "", ""], "3298": ["", "", "", ""], "3299": ["", "", "", ""], "3300": ["", "", "", ""], "3301": ["", "", "", ""], "3302": ["", "", "", ""], "3303": ["", "", "", ""], "3304": ["", "", "", ""], "3305": ["", "", "", ""], "3306": ["", "", "", ""], "3307": ["", "", "", ""], "3308": ["", "", "", ""], "3309": ["", "", "", ""], "3310": ["", "", "", ""], "3311": ["", "", "", ""], "3312": ["", "", "", ""], "3313": ["", "", "", ""], "3314": ["", "", "", ""], "3315": ["", "", "", ""], "3316": ["", "", "", ""], "3317": ["", "", "", ""], "3318": ["", "", "", ""], "3319": ["", "", "", ""], "3320": ["", "", "", ""], "3321": ["", "", "", ""], "3322": ["", "", "", ""], "3323": ["", "", "", ""], "3324": ["", "", "", ""], "3325": ["", "", "", ""], "3326": ["", "", "", ""], "3327": ["", "", "", ""], "3328": ["", "", "", ""], "3329": ["", "", "", ""], "3330": ["", "", "", ""], "3331": ["", "", "", ""], "3332": ["", "", "", ""], "3333": ["", "", "", ""], "3334": ["", "", "", ""], "3335": ["", "", "", ""], "3336": ["", "", "", ""], "3337": ["", "", "", ""], "3338": ["", "", "", ""], "3339": ["", "", "", ""], "3340": ["", "", "", ""], "3341": ["", "", "", ""], "3342": ["", "", "", ""], "3343": ["", "", "", ""], "3344": ["", "", "", ""], "3345": ["", "", "", ""], "3346": ["", "", "", ""], "3347": ["", "", "", ""], "3348": ["", "", "", ""], "3349": ["", "", "", ""], "3350": ["", "", "", ""], "3351": ["", "", "", ""], "3352": ["", "", "", ""], "3353": ["", "", "", ""], "3354": ["", "", "", ""], "3355": ["", "", "", ""], "3356": ["", "", "", ""], "3357": ["", "", "", ""], "3358": ["", "", "", ""], "3359": ["", "", "", ""], "3360": ["", "", "", ""], "3361": ["", "", "", ""], "3362": ["", "", "", ""], "3363": ["", "", "", ""], "3364": ["", "", "", ""], "3365": ["", "", "", ""], "3366": ["", "", "", ""], "3367": ["", "", "", ""], "3368": ["", "", "", ""], "3369": ["", "", "", ""], "3370": ["", "", "", ""], "3371": ["", "", "", ""], "3372": ["", "", "", ""], "3373": ["", "", "", ""], "3374": ["", "", "", ""], "3375": ["", "", "", ""], "3376": ["", "", "", ""], "3377": ["", "", "", ""], "3378": ["", "", "", ""], "3379": ["", "", "", ""], "3380": ["", "", "", ""], "3381": ["", "", "", ""], "3382": ["", "", "", ""], "3383": ["", "", "", ""], "3384": ["", "", "", ""], "3385": ["", "", "", ""], "3386": ["", "", "", ""], "3387": ["", "", "", ""], "3388": ["", "", "", ""], "3389": ["", "", "", ""], "3390": ["", "", "", ""], "3391": ["", "", "", ""], "3392": ["", "", "", ""], "3393": ["", "", "", ""], "3394": ["", "", "", ""], "3395": ["", "", "", ""], "3396": ["", "", "", ""], "3397": ["", "", "", ""], "3398": ["", "", "", ""], "3399": ["", "", "", ""], "3400": ["", "", "", ""], "3401": ["", "", "", ""], "3402": ["", "", "", ""], "3403": ["", "", "", ""], "3404": ["", "", "", ""], "3405": ["", "", "", ""], "3406": ["", "", "", ""], "3407": ["", "", "", ""], "3408": ["", "", "", ""], "3409": ["", "", "", ""], "3410": ["", "", "", ""], "3411": ["", "", "", ""], "3412": ["", "", "", ""], "3413": ["", "", "", ""], "3414": ["", "", "", ""], "3415": ["", "", "", ""], "3416": ["", "", "", ""], "3417": ["", "", "", ""], "3418": ["", "", "", ""], "3419": ["", "", "", ""], "3420": ["", "", "", ""], "3421": ["", "", "", ""], "3422": ["", "", "", ""], "3423": ["", "", "", ""], "3424": ["", "", "", ""], "3425": ["", "", "", ""], "3426": ["", "", "", ""], "3427": ["", "", "", ""], "3428": ["", "", "", ""], "3429": ["", "", "", ""], "3430": ["", "", "", ""], "3431": ["", "", "", ""], "3432": ["", "", "", ""], "3433": ["", "", "", ""], "3434": ["", "", "", ""], "3435": ["", "", "", ""], "3436": ["", "", "", ""], "3437": ["", "", "", ""], "3438": ["", "", "", ""], "3439": ["", "", "", ""], "3440": ["", "", "", ""], "3441": ["", "", "", ""], "3442": ["", "", "", ""], "3443": ["", "", "", ""], "3444": ["", "", "", ""], "3445": ["", "", "", ""], "3446": ["", "", "", ""], "3447": ["", "", "", ""], "3448": ["", "", "", ""], "3449": ["", "", "", ""], "3450": ["", "", "", ""], "3451": ["", "", "", ""], "3452": ["", "", "", ""], "3453": ["", "", "", ""], "3454": ["", "", "", ""], "3455": ["", "", "", ""], "3456": ["", "", "", ""], "3457": ["", "", "", ""], "3458": ["", "", "", ""], "3459": ["", "", "", ""], "3460": ["", "", "", ""], "3461": ["", "", "", ""], "3462": ["", "", "", ""], "3463": ["", "", "", ""], "3464": ["", "", "", ""], "3465": ["", "", "", ""], "3466": ["", "", "", ""], "3467": ["", "", "", ""], "3468": ["", "", "", ""], "3469": ["", "", "", ""], "3470": ["", "", "", ""], "3471": ["", "", "", ""], "3472": ["", "", "", ""], "3473": ["", "", "", ""], "3474": ["", "", "", ""], "3475": ["", "", "", ""], "3476": ["", "", "", ""], "3477": ["", "", "", ""], "3478": ["", "", "", ""], "3479": ["", "", "", ""], "3480": ["", "", "", ""], "3481": ["", "", "", ""], "3482": ["", "", "", ""], "3483": ["", "", "", ""], "3484": ["", "", "", ""], "3485": ["", "", "", ""], "3486": ["", "", "", ""], "3487": ["", "", "", ""], "3488": ["", "", "", ""], "3489": ["", "", "", ""], "3490": ["", "", "", ""], "3491": ["", "", "", ""], "3492": ["", "", "", ""], "3493": ["", "", "", ""], "3494": ["", "", "", ""], "3495": ["", "", "", ""], "3496": ["", "", "", ""], "3497": ["", "", "", ""], "3498": ["", "", "", ""], "3499": ["", "", "", ""], "3500": ["", "", "", ""], "3501": ["", "", "", ""], "3502": ["", "", "", ""], "3503": ["", "", "", ""], "3504": ["", "", "", ""], "3505": ["", "", "", ""], "3506": ["", "", "", ""], "3507": ["", "", "", ""], "3508": ["", "", "", ""], "3509": ["", "", "", ""], "3510": ["", "", "", ""], "3511": ["", "", "", ""], "3512": ["", "", "", ""], "3513": ["", "", "", ""], "3514": ["", "", "", ""], "3515": ["", "", "", ""], "3516": ["", "", "", ""], "3517": ["", "", "", ""], "3518": ["", "", "", ""], "3519": ["", "", "", ""], "3520": ["", "", "", ""], "3521": ["", "", "", ""], "3522": ["", "", "", ""], "3523": ["", "", "", ""], "3524": ["", "", "", ""], "3525": ["", "", "", ""], "3526": ["", "", "", ""], "3527": ["", "", "", ""], "3528": ["", "", "", ""], "3529": ["", "", "", ""], "3530": ["", "", "", ""], "3531": ["", "", "", ""], "3532": ["", "", "", ""], "3533": ["", "", "", ""], "3534": ["", "", "", ""], "3535": ["", "", "", ""], "3536": ["", "", "", ""], "3537": ["", "", "", ""], "3538": ["", "", "", ""], "3539": ["", "", "", ""], "3540": ["", "", "", ""], "3541": ["", "", "", ""], "3542": ["", "", "", ""], "3543": ["", "", "", ""], "3544": ["", "", "", ""], "3545": ["", "", "", ""], "3546": ["", "", "", ""], "3547": ["", "", "", ""], "3548": ["", "", "", ""], "3549": ["", "", "", ""], "3550": ["", "", "", ""], "3551": ["", "", "", ""], "3552": ["", "", "", ""], "3553": ["", "", "", ""], "3554": ["", "", "", ""], "3555": ["", "", "", ""], "3556": ["", "", "", ""], "3557": ["", "", "", ""], "3558": ["", "", "", ""], "3559": ["", "", "", ""], "3560": ["", "", "", ""], "3561": ["", "", "", ""], "3562": ["", "", "", ""], "3563": ["", "", "", ""], "3564": ["", "", "", ""], "3565": ["", "", "", ""], "3566": ["", "", "", ""], "3567": ["", "", "", ""], "3568": ["", "", "", ""], "3569": ["", "", "", ""], "3570": ["", "", "", ""], "3571": ["", "", "", ""], "3572": ["", "", "", ""], "3573": ["", "", "", ""], "3574": ["", "", "", ""], "3575": ["", "", "", ""], "3576": ["", "", "", ""], "3577": ["", "", "", ""], "3578": ["", "", "", ""], "3579": ["", "", "", ""], "3580": ["", "", "", ""], "3581": ["", "", "", ""], "3582": ["", "", "", ""], "3583": ["", "", "", ""], "3584": ["", "", "", ""], "3585": ["", "", "", ""], "3586": ["", "", "", ""], "3587": ["", "", "", ""], "3588": ["", "", "", ""], "3589": ["", "", "", ""], "3590": ["", "", "", ""], "3591": ["", "", "", ""], "3592": ["", "", "", ""], "3593": ["", "", "", ""], "3594": ["", "", "", ""], "3595": ["", "", "", ""], "3596": ["", "", "", ""], "3597": ["", "", "", ""], "3598": ["", "", "", ""], "3599": ["", "", "", ""], "3600": ["", "", "", ""], "3601": ["", "", "", ""], "3602": ["", "", "", ""], "3603": ["", "", "", ""], "3604": ["", "", "", ""], "3605": ["", "", "", ""], "3606": ["", "", "", ""], "3607": ["", "", "", ""], "3608": ["", "", "", ""], "3609": ["", "", "", ""], "3610": ["", "", "", ""], "3611": ["", "", "", ""], "3612": ["", "", "", ""], "3613": ["", "", "", ""], "3614": ["", "", "", ""], "3615": ["", "", "", ""], "3616": ["", "", "", ""], "3617": ["", "", "", ""], "3618": ["", "", "", ""], "3619": ["", "", "", ""], "3620": ["", "", "", ""], "3621": ["", "", "", ""], "3622": ["", "", "", ""], "3623": ["", "", "", ""], "3624": ["", "", "", ""], "3625": ["", "", "", ""], "3626": ["", "", "", ""], "3627": ["", "", "", ""], "3628": ["", "", "", ""], "3629": ["", "", "", ""], "3630": ["", "", "", ""], "3631": ["", "", "", ""], "3632": ["", "", "", ""], "3633": ["", "", "", ""], "3634": ["", "", "", ""], "3635": ["", "", "", ""], "3636": ["", "", "", ""], "3637": ["", "", "", ""], "3638": ["", "", "", ""], "3639": ["", "", "", ""], "3640": ["", "", "", ""], "3641": ["", "", "", ""], "3642": ["", "", "", ""], "3643": ["", "", "", ""], "3644": ["", "", "", ""], "3645": ["", "", "", ""], "3646": ["", "", "", ""], "3647": ["", "", "", ""], "3648": ["", "", "", ""], "3649": ["", "", "", ""], "3650": ["", "", "", ""], "3651": ["", "", "", ""], "3652": ["", "", "", ""], "3653": ["", "", "", ""], "3654": ["", "", "", ""], "3655": ["", "", "", ""], "3656": ["", "", "", ""], "3657": ["", "", "", ""], "3658": ["", "", "", ""], "3659": ["", "", "", ""], "3660": ["", "", "", ""], "3661": ["", "", "", ""], "3662": ["", "", "", ""], "3663": ["", "", "", ""], "3664": ["", "", "", ""], "3665": ["", "", "", ""], "3666": ["", "", "", ""], "3667": ["", "", "", ""], "3668": ["", "", "", ""], "3669": ["", "", "", ""], "3670": ["", "", "", ""], "3671": ["", "", "", ""], "3672": ["", "", "", ""], "3673": ["", "", "", ""], "3674": ["", "", "", ""], "3675": ["", "", "", ""], "3676": ["", "", "", ""], "3677": ["", "", "", ""], "3678": ["", "", "", ""], "3679": ["", "", "", ""], "3680": ["", "", "", ""], "3681": ["", "", "", ""], "3682": ["", "", "", ""], "3683": ["", "", "", ""], "3684": ["", "", "", ""], "3685": ["", "", "", ""], "3686": ["", "", "", ""], "3687": ["", "", "", ""], "3688": ["", "", "", ""], "3689": ["", "", "", ""], "3690": ["", "", "", ""], "3691": ["", "", "", ""], "3692": ["", "", "", ""], "3693": ["", "", "", ""], "3694": ["", "", "", ""], "3695": ["", "", "", ""], "3696": ["", "", "", ""], "3697": ["", "", "", ""], "3698": ["", "", "", ""], "3699": ["", "", "", ""], "3700": ["", "", "", ""], "3701": ["", "", "", ""], "3702": ["", "", "", ""], "3703": ["", "", "", ""], "3704": ["", "", "", ""], "3705": ["", "", "", ""], "3706": ["", "", "", ""], "3707": ["", "", "", ""], "3708": ["", "", "", ""], "3709": ["", "", "", ""], "3710": ["", "", "", ""], "3711": ["", "", "", ""], "3712": ["", "", "", ""], "3713": ["", "", "", ""], "3714": ["", "", "", ""], "3715": ["", "", "", ""], "3716": ["", "", "", ""], "3717": ["", "", "", ""], "3718": ["", "", "", ""], "3719": ["", "", "", ""], "3720": ["", "", "", ""], "3721": ["", "", "", ""], "3722": ["", "", "", ""], "3723": ["", "", "", ""], "3724": ["", "", "", ""], "3725": ["", "", "", ""], "3726": ["", "", "", ""], "3727": ["", "", "", ""], "3728": ["", "", "", ""], "3729": ["", "", "", ""], "3730": ["", "", "", ""], "3731": ["", "", "", ""], "3732": ["", "", "", ""], "3733": ["", "", "", ""], "3734": ["", "", "", ""], "3735": ["", "", "", ""], "3736": ["", "", "", ""], "3737": ["", "", "", ""], "3738": ["", "", "", ""], "3739": ["", "", "", ""], "3740": ["", "", "", ""], "3741": ["", "", "", ""], "3742": ["", "", "", ""], "3743": ["", "", "", ""], "3744": ["", "", "", ""], "3745": ["", "", "", ""], "3746": ["", "", "", ""], "3747": ["", "", "", ""], "3748": ["", "", "", ""], "3749": ["", "", "", ""], "3750": ["", "", "", ""], "3751": ["", "", "", ""], "3752": ["", "", "", ""], "3753": ["", "", "", ""], "3754": ["", "", "", ""], "3755": ["", "", "", ""], "3756": ["", "", "", ""], "3757": ["", "", "", ""], "3758": ["", "", "", ""], "3759": ["", "", "", ""], "3760": ["", "", "", ""], "3761": ["", "", "", ""], "3762": ["", "", "", ""], "3763": ["", "", "", ""], "3764": ["", "", "", ""], "3765": ["", "", "", ""], "3766": ["", "", "", ""], "3767": ["", "", "", ""], "3768": ["", "", "", ""], "3769": ["", "", "", ""], "3770": ["", "", "", ""], "3771": ["", "", "", ""], "3772": ["", "", "", ""], "3773": ["", "", "", ""], "3774": ["", "", "", ""], "3775": ["", "", "", ""], "3776": ["", "", "", ""], "3777": ["", "", "", ""], "3778": ["", "", "", ""], "3779": ["", "", "", ""], "3780": ["", "", "", ""], "3781": ["", "", "", ""], "3782": ["", "", "", ""], "3783": ["", "", "", ""], "3784": ["", "", "", ""], "3785": ["", "", "", ""], "3786": ["", "", "", ""], "3787": ["", "", "", ""], "3788": ["", "", "", ""], "3789": ["", "", "", ""], "3790": ["", "", "", ""], "3791": ["", "", "", ""], "3792": ["", "", "", ""], "3793": ["", "", "", ""], "3794": ["", "", "", ""], "3795": ["", "", "", ""], "3796": ["", "", "", ""], "3797": ["", "", "", ""], "3798": ["", "", "", ""], "3799": ["", "", "", ""], "3800": ["", "", "", ""], "3801": ["", "", "", ""], "3802": ["", "", "", ""], "3803": ["", "", "", ""], "3804": ["", "", "", ""], "3805": ["", "", "", ""], "3806": ["", "", "", ""], "3807": ["", "", "", ""], "3808": ["", "", "", ""], "3809": ["", "", "", ""], "3810": ["", "", "", ""], "3811": ["", "", "", ""], "3812": ["", "", "", ""], "3813": ["", "", "", ""], "3814": ["", "", "", ""], "3815": ["", "", "", ""], "3816": ["", "", "", ""], "3817": ["", "", "", ""], "3818": ["", "", "", ""], "3819": ["", "", "", ""], "3820": ["", "", "", ""], "3821": ["", "", "", ""], "3822": ["", "", "", ""], "3823": ["", "", "", ""], "3824": ["", "", "", ""], "3825": ["", "", "", ""], "3826": ["", "", "", ""], "3827": ["", "", "", ""], "3828": ["", "", "", ""], "3829": ["", "", "", ""], "3830": ["", "", "", ""], "3831": ["", "", "", ""], "3832": ["", "", "", ""], "3833": ["", "", "", ""], "3834": ["", "", "", ""], "3835": ["", "", "", ""], "3836": ["", "", "", ""], "3837": ["", "", "", ""], "3838": ["", "", "", ""], "3839": ["", "", "", ""], "3840": ["", "", "", ""], "3841": ["", "", "", ""], "3842": ["", "", "", ""], "3843": ["", "", "", ""], "3844": ["", "", "", ""], "3845": ["", "", "", ""], "3846": ["", "", "", ""], "3847": ["", "", "", ""], "3848": ["", "", "", ""], "3849": ["", "", "", ""], "3850": ["", "", "", ""], "3851": ["", "", "", ""], "3852": ["", "", "", ""], "3853": ["", "", "", ""], "3854": ["", "", "", ""], "3855": ["", "", "", ""], "3856": ["", "", "", ""], "3857": ["", "", "", ""], "3858": ["", "", "", ""], "3859": ["", "", "", ""], "3860": ["", "", "", ""], "3861": ["", "", "", ""], "3862": ["", "", "", ""], "3863": ["", "", "", ""], "3864": ["", "", "", ""], "3865": ["", "", "", ""], "3866": ["", "", "", ""], "3867": ["", "", "", ""], "3868": ["", "", "", ""], "3869": ["", "", "", ""], "3870": ["", "", "", ""], "3871": ["", "", "", ""], "3872": ["", "", "", ""], "3873": ["", "", "", ""], "3874": ["", "", "", ""], "3875": ["", "", "", ""], "3876": ["", "", "", ""], "3877": ["", "", "", ""], "3878": ["", "", "", ""], "3879": ["", "", "", ""], "3880": ["", "", "", ""], "3881": ["", "", "", ""], "3882": ["", "", "", ""], "3883": ["", "", "", ""], "3884": ["", "", "", ""], "3885": ["", "", "", ""], "3886": ["", "", "", ""], "3887": ["", "", "", ""], "3888": ["", "", "", ""], "3889": ["", "", "", ""], "3890": ["", "", "", ""], "3891": ["", "", "", ""], "3892": ["", "", "", ""], "3893": ["", "", "", ""], "3894": ["", "", "", ""], "3895": ["", "", "", ""], "3896": ["", "", "", ""], "3897": ["", "", "", ""], "3898": ["", "", "", ""], "3899": ["", "", "", ""], "3900": ["", "", "", ""], "3901": ["", "", "", ""], "3902": ["", "", "", ""], "3903": ["", "", "", ""], "3904": ["", "", "", ""], "3905": ["", "", "", ""], "3906": ["", "", "", ""], "3907": ["", "", "", ""], "3908": ["", "", "", ""], "3909": ["", "", "", ""], "3910": ["", "", "", ""], "3911": ["", "", "", ""], "3912": ["", "", "", ""], "3913": ["", "", "", ""], "3914": ["", "", "", ""], "3915": ["", "", "", ""], "3916": ["", "", "", ""], "3917": ["", "", "", ""], "3918": ["", "", "", ""], "3919": ["", "", "", ""], "3920": ["", "", "", ""], "3921": ["", "", "", ""], "3922": ["", "", "", ""], "3923": ["", "", "", ""], "3924": ["", "", "", ""], "3925": ["", "", "", ""], "3926": ["", "", "", ""], "3927": ["", "", "", ""], "3928": ["", "", "", ""], "3929": ["", "", "", ""], "3930": ["", "", "", ""], "3931": ["", "", "", ""], "3932": ["", "", "", ""], "3933": ["", "", "", ""], "3934": ["", "", "", ""], "3935": ["", "", "", ""], "3936": ["", "", "", ""], "3937": ["", "", "", ""], "3938": ["", "", "", ""], "3939": ["", "", "", ""], "3940": ["", "", "", ""], "3941": ["", "", "", ""], "3942": ["", "", "", ""], "3943": ["", "", "", ""], "3944": ["", "", "", ""], "3945": ["", "", "", ""], "3946": ["", "", "", ""], "3947": ["", "", "", ""], "3948": ["", "", "", ""], "3949": ["", "", "", ""], "3950": ["", "", "", ""], "3951": ["", "", "", ""], "3952": ["", "", "", ""], "3953": ["", "", "", ""], "3954": ["", "", "", ""], "3955": ["", "", "", ""], "3956": ["", "", "", ""], "3957": ["", "", "", ""], "3958": ["", "", "", ""], "3959": ["", "", "", ""], "3960": ["", "", "", ""], "3961": ["", "", "", ""], "3962": ["", "", "", ""], "3963": ["", "", "", ""], "3964": ["", "", "", ""], "3965": ["", "", "", ""], "3966": ["", "", "", ""], "3967": ["", "", "", ""], "3968": ["", "", "", ""], "3969": ["", "", "", ""], "3970": ["", "", "", ""], "3971": ["", "", "", ""], "3972": ["", "", "", ""], "3973": ["", "", "", ""], "3974": ["", "", "", ""], "3975": ["", "", "", ""], "3976": ["", "", "", ""], "3977": ["", "", "", ""], "3978": ["", "", "", ""], "3979": ["", "", "", ""], "3980": ["", "", "", ""], "3981": ["", "", "", ""], "3982": ["", "", "", ""], "3983": ["", "", "", ""], "3984": ["", "", "", ""], "3985": ["", "", "", ""], "3986": ["", "", "", ""], "3987": ["", "", "", ""], "3988": ["", "", "", ""], "3989": ["", "", "", ""], "3990": ["", "", "", ""], "3991": ["", "", "", ""], "3992": ["", "", "", ""], "3993": ["", "", "", ""], "3994": ["", "", "", ""], "3995": ["", "", "", ""], "3996": ["", "", "", ""], "3997": ["", "", "", ""], "3998": ["", "", "", ""], "3999": ["", "", "", ""], "4000": ["", "", "", ""], "4001": ["", "", "", ""], "4002": ["", "", "", ""], "4003": ["", "", "", ""], "4004": ["", "", "", ""], "4005": ["", "", "", ""], "4006": ["", "", "", ""], "4007": ["", "", "", ""], "4008": ["", "", "", ""], "4009": ["", "", "", ""], "4010": ["", "", "", ""], "4011": ["", "", "", ""], "4012": ["", "", "", ""], "4013": ["", "", "", ""], "4014": ["", "", "", ""], "4015": ["", "", "", ""], "4016": ["", "", "", ""], "4017": ["", "", "", ""], "4018": ["", "", "", ""], "4019": ["", "", "", ""], "4020": ["", "", "", ""], "4021": ["", "", "", ""], "4022": ["", "", "", ""], "4023": ["", "", "", ""], "4024": ["", "", "", ""], "4025": ["", "", "", ""], "4026": ["", "", "", ""], "4027": ["", "", "", ""], "4028": ["", "", "", ""], "4029": ["", "", "", ""], "4030": ["", "", "", ""], "4031": ["", "", "", ""], "4032": ["", "", "", ""], "4033": ["", "", "", ""], "4034": ["", "", "", ""], "4035": ["", "", "", ""], "4036": ["", "", "", ""], "4037": ["", "", "", ""], "4038": ["", "", "", ""], "4039": ["", "", "", ""], "4040": ["", "", "", ""], "4041": ["", "", "", ""], "4042": ["", "", "", ""], "4043": ["", "", "", ""], "4044": ["", "", "", ""], "4045": ["", "", "", ""], "4046": ["", "", "", ""], "4047": ["", "", "", ""], "4048": ["", "", "", ""], "4049": ["", "", "", ""], "4050": ["", "", "", ""], "4051": ["", "", "", ""], "4052": ["", "", "", ""], "4053": ["", "", "", ""], "4054": ["", "", "", ""], "4055": ["", "", "", ""], "4056": ["", "", "", ""], "4057": ["", "", "", ""], "4058": ["", "", "", ""], "4059": ["", "", "", ""], "4060": ["", "", "", ""], "4061": ["", "", "", ""], "4062": ["", "", "", ""], "4063": ["", "", "", ""], "4064": ["", "", "", ""], "4065": ["", "", "", ""], "4066": ["", "", "", ""], "4067": ["", "", "", ""], "4068": ["", "", "", ""], "4069": ["", "", "", ""], "4070": ["", "", "", ""], "4071": ["", "", "", ""], "4072": ["", "", "", ""], "4073": ["", "", "", ""], "4074": ["", "", "", ""], "4075": ["", "", "", ""], "4076": ["", "", "", ""], "4077": ["", "", "", ""], "4078": ["", "", "", ""], "4079": ["", "", "", ""], "4080": ["", "", "", ""], "4081": ["", "", "", ""], "4082": ["", "", "", ""], "4083": ["", "", "", ""], "4084": ["", "", "", ""], "4085": ["", "", "", ""], "4086": ["", "", "", ""], "4087": ["", "", "", ""], "4088": ["", "", "", ""], "4089": ["", "", "", ""], "4090": ["", "", "", ""], "4091": ["", "", "", ""], "4092": ["", "", "", ""], "4093": ["", "", "", ""], "4094": ["", "", "", ""], "4095": ["", "", "", ""], "4096": ["", "", "", ""], "4097": ["", "", "", ""], "4098": ["", "", "", ""], "4099": ["", "", "", ""], "4100": ["", "", "", ""], "4101": ["", "", "", ""], "4102": ["", "", "", ""], "4103": ["", "", "", ""], "4104": ["", "", "", ""], "4105": ["", "", "", ""], "4106": ["", "", "", ""], "4107": ["", "", "", ""], "4108": ["", "", "", ""], "4109": ["", "", "", ""], "4110": ["", "", "", ""], "4111": ["", "", "", ""], "4112": ["", "", "", ""], "4113": ["", "", "", ""], "4114": ["", "", "", ""], "4115": ["", "", "", ""], "4116": ["", "", "", ""], "4117": ["", "", "", ""], "4118": ["", "", "", ""], "4119": ["", "", "", ""], "4120": ["", "", "", ""], "4121": ["", "", "", ""], "4122": ["", "", "", ""], "4123": ["", "", "", ""], "4124": ["", "", "", ""], "4125": ["", "", "", ""], "4126": ["", "", "", ""], "4127": ["", "", "", ""], "4128": ["", "", "", ""], "4129": ["", "", "", ""], "4130": ["", "", "", ""], "4131": ["", "", "", ""], "4132": ["", "", "", ""], "4133": ["", "", "", ""], "4134": ["", "", "", ""], "4135": ["", "", "", ""], "4136": ["", "", "", ""], "4137": ["", "", "", ""], "4138": ["", "", "", ""], "4139": ["", "", "", ""], "4140": ["", "", "", ""], "4141": ["", "", "", ""], "4142": ["", "", "", ""], "4143": ["", "", "", ""], "4144": ["", "", "", ""], "4145": ["", "", "", ""], "4146": ["", "", "", ""], "4147": ["", "", "", ""], "4148": ["", "", "", ""], "4149": ["", "", "", ""], "4150": ["", "", "", ""], "4151": ["", "", "", ""], "4152": ["", "", "", ""], "4153": ["", "", "", ""], "4154": ["", "", "", ""], "4155": ["", "", "", ""], "4156": ["", "", "", ""], "4157": ["", "", "", ""], "4158": ["", "", "", ""], "4159": ["", "", "", ""], "4160": ["", "", "", ""], "4161": ["", "", "", ""], "4162": ["", "", "", ""], "4163": ["", "", "", ""], "4164": ["", "", "", ""], "4165": ["", "", "", ""], "4166": ["", "", "", ""], "4167": ["", "", "", ""], "4168": ["", "", "", ""], "4169": ["", "", "", ""], "4170": ["", "", "", ""], "4171": ["", "", "", ""], "4172": ["", "", "", ""], "4173": ["", "", "", ""], "4174": ["", "", "", ""], "4175": ["", "", "", ""], "4176": ["", "", "", ""], "4177": ["", "", "", ""], "4178": ["", "", "", ""], "4179": ["", "", "", ""], "4180": ["", "", "", ""], "4181": ["", "", "", ""], "4182": ["", "", "", ""], "4183": ["", "", "", ""], "4184": ["", "", "", ""], "4185": ["", "", "", ""], "4186": ["", "", "", ""], "4187": ["", "", "", ""], "4188": ["", "", "", ""], "4189": ["", "", "", ""], "4190": ["", "", "", ""], "4191": ["", "", "", ""], "4192": ["", "", "", ""], "4193": ["", "", "", ""], "4194": ["", "", "", ""], "4195": ["", "", "", ""], "4196": ["", "", "", ""], "4197": ["", "", "", ""], "4198": ["", "", "", ""], "4199": ["", "", "", ""], "4200": ["", "", "", ""], "4201": ["", "", "", ""], "4202": ["", "", "", ""], "4203": ["", "", "", ""], "4204": ["", "", "", ""], "4205": ["", "", "", ""], "4206": ["", "", "", ""], "4207": ["", "", "", ""], "4208": ["", "", "", ""], "4209": ["", "", "", ""], "4210": ["", "", "", ""], "4211": ["", "", "", ""], "4212": ["", "", "", ""], "4213": ["", "", "", ""], "4214": ["", "", "", ""], "4215": ["", "", "", ""], "4216": ["", "", "", ""], "4217": ["", "", "", ""], "4218": ["", "", "", ""], "4219": ["", "", "", ""], "4220": ["", "", "", ""], "4221": ["", "", "", ""], "4222": ["", "", "", ""], "4223": ["", "", "", ""], "4224": ["", "", "", ""], "4225": ["", "", "", ""], "4226": ["", "", "", ""], "4227": ["", "", "", ""], "4228": ["", "", "", ""], "4229": ["", "", "", ""], "4230": ["", "", "", ""], "4231": ["", "", "", ""], "4232": ["", "", "", ""], "4233": ["", "", "", ""], "4234": ["", "", "", ""], "4235": ["", "", "", ""], "4236": ["", "", "", ""], "4237": ["", "", "", ""], "4238": ["", "", "", ""], "4239": ["", "", "", ""], "4240": ["", "", "", ""], "4241": ["", "", "", ""], "4242": ["", "", "", ""], "4243": ["", "", "", ""], "4244": ["", "", "", ""], "4245": ["", "", "", ""], "4246": ["", "", "", ""], "4247": ["", "", "", ""], "4248": ["", "", "", ""], "4249": ["", "", "", ""], "4250": ["", "", "", ""], "4251": ["", "", "", ""], "4252": ["", "", "", ""], "4253": ["", "", "", ""], "4254": ["", "", "", ""], "4255": ["", "", "", ""], "4256": ["", "", "", ""], "4257": ["", "", "", ""], "4258": ["", "", "", ""], "4259": ["", "", "", ""], "4260": ["", "", "", ""], "4261": ["", "", "", ""], "4262": ["", "", "", ""], "4263": ["", "", "", ""], "4264": ["", "", "", ""], "4265": ["", "", "", ""], "4266": ["", "", "", ""], "4267": ["", "", "", ""], "4268": ["", "", "", ""], "4269": ["", "", "", ""], "4270": ["", "", "", ""], "4271": ["", "", "", ""], "4272": ["", "", "", ""], "4273": ["", "", "", ""], "4274": ["", "", "", ""], "4275": ["", "", "", ""], "4276": ["", "", "", ""], "4277": ["", "", "", ""], "4278": ["", "", "", ""], "4279": ["", "", "", ""], "4280": ["", "", "", ""], "4281": ["", "", "", ""], "4282": ["", "", "", ""], "4283": ["", "", "", ""], "4284": ["", "", "", ""], "4285": ["", "", "", ""], "4286": ["", "", "", ""], "4287": ["", "", "", ""], "4288": ["", "", "", ""], "4289": ["", "", "", ""], "4290": ["", "", "", ""], "4291": ["", "", "", ""], "4292": ["", "", "", ""], "4293": ["", "", "", ""], "4294": ["", "", "", ""], "4295": ["", "", "", ""], "4296": ["", "", "", ""], "4297": ["", "", "", ""], "4298": ["", "", "", ""], "4299": ["", "", "", ""], "4300": ["", "", "", ""], "4301": ["", "", "", ""], "4302": ["", "", "", ""], "4303": ["", "", "", ""], "4304": ["", "", "", ""], "4305": ["", "", "", ""], "4306": ["", "", "", ""], "4307": ["", "", "", ""], "4308": ["", "", "", ""], "4309": ["", "", "", ""], "4310": ["", "", "", ""], "4311": ["", "", "", ""], "4312": ["", "", "", ""], "4313": ["", "", "", ""], "4314": ["", "", "", ""], "4315": ["", "", "", ""], "4316": ["", "", "", ""], "4317": ["", "", "", ""], "4318": ["", "", "", ""], "4319": ["", "", "", ""], "4320": ["", "", "", ""], "4321": ["", "", "", ""], "4322": ["", "", "", ""], "4323": ["", "", "", ""], "4324": ["", "", "", ""], "4325": ["", "", "", ""], "4326": ["", "", "", ""], "4327": ["", "", "", ""], "4328": ["", "", "", ""], "4329": ["", "", "", ""], "4330": ["", "", "", ""], "4331": ["", "", "", ""], "4332": ["", "", "", ""], "4333": ["", "", "", ""], "4334": ["", "", "", ""], "4335": ["", "", "", ""], "4336": ["", "", "", ""], "4337": ["", "", "", ""], "4338": ["", "", "", ""], "4339": ["", "", "", ""], "4340": ["", "", "", ""], "4341": ["", "", "", ""], "4342": ["", "", "", ""], "4343": ["", "", "", ""], "4344": ["", "", "", ""], "4345": ["", "", "", ""], "4346": ["", "", "", ""], "4347": ["", "", "", ""], "4348": ["", "", "", ""], "4349": ["", "", "", ""], "4350": ["", "", "", ""], "4351": ["", "", "", ""], "4352": ["", "", "", ""], "4353": ["", "", "", ""], "4354": ["", "", "", ""], "4355": ["", "", "", ""], "4356": ["", "", "", ""], "4357": ["", "", "", ""], "4358": ["", "", "", ""], "4359": ["", "", "", ""], "4360": ["", "", "", ""], "4361": ["", "", "", ""], "4362": ["", "", "", ""], "4363": ["", "", "", ""], "4364": ["", "", "", ""], "4365": ["", "", "", ""], "4366": ["", "", "", ""], "4367": ["", "", "", ""], "4368": ["", "", "", ""], "4369": ["", "", "", ""], "4370": ["", "", "", ""], "4371": ["", "", "", ""], "4372": ["", "", "", ""], "4373": ["", "", "", ""], "4374": ["", "", "", ""], "4375": ["", "", "", ""], "4376": ["", "", "", ""], "4377": ["", "", "", ""], "4378": ["", "", "", ""], "4379": ["", "", "", ""], "4380": ["", "", "", ""], "4381": ["", "", "", ""], "4382": ["", "", "", ""], "4383": ["", "", "", ""], "4384": ["", "", "", ""], "4385": ["", "", "", ""], "4386": ["", "", "", ""], "4387": ["", "", "", ""], "4388": ["", "", "", ""], "4389": ["", "", "", ""], "4390": ["", "", "", ""], "4391": ["", "", "", ""], "4392": ["", "", "", ""], "4393": ["", "", "", ""], "4394": ["", "", "", ""], "4395": ["", "", "", ""], "4396": ["", "", "", ""], "4397": ["", "", "", ""], "4398": ["", "", "", ""], "4399": ["", "", "", ""], "4400": ["", "", "", ""], "4401": ["", "", "", ""], "4402": ["", "", "", ""], "4403": ["", "", "", ""], "4404": ["", "", "", ""], "4405": ["", "", "", ""], "4406": ["", "", "", ""], "4407": ["", "", "", ""], "4408": ["", "", "", ""], "4409": ["", "", "", ""], "4410": ["", "", "", ""], "4411": ["", "", "", ""], "4412": ["", "", "", ""], "4413": ["", "", "", ""], "4414": ["", "", "", ""], "4415": ["", "", "", ""], "4416": ["", "", "", ""], "4417": ["", "", "", ""], "4418": ["", "", "", ""], "4419": ["", "", "", ""], "4420": ["", "", "", ""], "4421": ["", "", "", ""], "4422": ["", "", "", ""], "4423": ["", "", "", ""], "4424": ["", "", "", ""], "4425": ["", "", "", ""], "4426": ["", "", "", ""], "4427": ["", "", "", ""], "4428": ["", "", "", ""], "4429": ["", "", "", ""], "4430": ["", "", "", ""], "4431": ["", "", "", ""], "4432": ["", "", "", ""], "4433": ["", "", "", ""], "4434": ["", "", "", ""], "4435": ["", "", "", ""], "4436": ["", "", "", ""], "4437": ["", "", "", ""], "4438": ["", "", "", ""], "4439": ["", "", "", ""], "4440": ["", "", "", ""], "4441": ["", "", "", ""], "4442": ["", "", "", ""], "4443": ["", "", "", ""], "4444": ["", "", "", ""], "4445": ["", "", "", ""], "4446": ["", "", "", ""], "4447": ["", "", "", ""], "4448": ["", "", "", ""], "4449": ["", "", "", ""], "4450": ["", "", "", ""], "4451": ["", "", "", ""], "4452": ["", "", "", ""], "4453": ["", "", "", ""], "4454": ["", "", "", ""], "4455": ["", "", "", ""], "4456": ["", "", "", ""], "4457": ["", "", "", ""], "4458": ["", "", "", ""], "4459": ["", "", "", ""], "4460": ["", "", "", ""], "4461": ["", "", "", ""], "4462": ["", "", "", ""], "4463": ["", "", "", ""], "4464": ["", "", "", ""], "4465": ["", "", "", ""], "4466": ["", "", "", ""], "4467": ["", "", "", ""], "4468": ["", "", "", ""], "4469": ["", "", "", ""], "4470": ["", "", "", ""], "4471": ["", "", "", ""], "4472": ["", "", "", ""], "4473": ["", "", "", ""], "4474": ["", "", "", ""], "4475": ["", "", "", ""], "4476": ["", "", "", ""], "4477": ["", "", "", ""], "4478": ["", "", "", ""], "4479": ["", "", "", ""], "4480": ["", "", "", ""], "4481": ["", "", "", ""], "4482": ["", "", "", ""], "4483": ["", "", "", ""], "4484": ["", "", "", ""], "4485": ["", "", "", ""], "4486": ["", "", "", ""], "4487": ["", "", "", ""], "4488": ["", "", "", ""], "4489": ["", "", "", ""], "4490": ["", "", "", ""], "4491": ["", "", "", ""], "4492": ["", "", "", ""], "4493": ["", "", "", ""], "4494": ["", "", "", ""], "4495": ["", "", "", ""], "4496": ["", "", "", ""], "4497": ["", "", "", ""], "4498": ["", "", "", ""], "4499": ["", "", "", ""], "4500": ["", "", "", ""], "4501": ["", "", "", ""], "4502": ["", "", "", ""], "4503": ["", "", "", ""], "4504": ["", "", "", ""], "4505": ["", "", "", ""], "4506": ["", "", "", ""], "4507": ["", "", "", ""], "4508": ["", "", "", ""], "4509": ["", "", "", ""], "4510": ["", "", "", ""], "4511": ["", "", "", ""], "4512": ["", "", "", ""], "4513": ["", "", "", ""], "4514": ["", "", "", ""], "4515": ["", "", "", ""], "4516": ["", "", "", ""], "4517": ["", "", "", ""], "4518": ["", "", "", ""], "4519": ["", "", "", ""], "4520": ["", "", "", ""], "4521": ["", "", "", ""], "4522": ["", "", "", ""], "4523": ["", "", "", ""], "4524": ["", "", "", ""], "4525": ["", "", "", ""], "4526": ["", "", "", ""], "4527": ["", "", "", ""], "4528": ["", "", "", ""], "4529": ["", "", "", ""], "4530": ["", "", "", ""], "4531": ["", "", "", ""], "4532": ["", "", "", ""], "4533": ["", "", "", ""], "4534": ["", "", "", ""], "4535": ["", "", "", ""], "4536": ["", "", "", ""], "4537": ["", "", "", ""], "4538": ["", "", "", ""], "4539": ["", "", "", ""], "4540": ["", "", "", ""], "4541": ["", "", "", ""], "4542": ["", "", "", ""], "4543": ["", "", "", ""], "4544": ["", "", "", ""], "4545": ["", "", "", ""], "4546": ["", "", "", ""], "4547": ["", "", "", ""], "4548": ["", "", "", ""], "4549": ["", "", "", ""], "4550": ["", "", "", ""], "4551": ["", "", "", ""], "4552": ["", "", "", ""], "4553": ["", "", "", ""], "4554": ["", "", "", ""], "4555": ["", "", "", ""], "4556": ["", "", "", ""], "4557": ["", "", "", ""], "4558": ["", "", "", ""], "4559": ["", "", "", ""], "4560": ["", "", "", ""], "4561": ["", "", "", ""], "4562": ["", "", "", ""], "4563": ["", "", "", ""], "4564": ["", "", "", ""], "4565": ["", "", "", ""], "4566": ["", "", "", ""], "4567": ["", "", "", ""], "4568": ["", "", "", ""], "4569": ["", "", "", ""], "4570": ["", "", "", ""], "4571": ["", "", "", ""], "4572": ["", "", "", ""], "4573": ["", "", "", ""], "4574": ["", "", "", ""], "4575": ["", "", "", ""], "4576": ["", "", "", ""], "4577": ["", "", "", ""], "4578": ["", "", "", ""], "4579": ["", "", "", ""], "4580": ["", "", "", ""], "4581": ["", "", "", ""], "4582": ["", "", "", ""], "4583": ["", "", "", ""], "4584": ["", "", "", ""], "4585": ["", "", "", ""], "4586": ["", "", "", ""], "4587": ["", "", "", ""], "4588": ["", "", "", ""], "4589": ["", "", "", ""], "4590": ["", "", "", ""], "4591": ["", "", "", ""], "4592": ["", "", "", ""], "4593": ["", "", "", ""], "4594": ["", "", "", ""], "4595": ["", "", "", ""], "4596": ["", "", "", ""], "4597": ["", "", "", ""], "4598": ["", "", "", ""], "4599": ["", "", "", ""], "4600": ["", "", "", ""], "4601": ["", "", "", ""], "4602": ["", "", "", ""], "4603": ["", "", "", ""], "4604": ["", "", "", ""], "4605": ["", "", "", ""], "4606": ["", "", "", ""], "4607": ["", "", "", ""], "4608": ["", "", "", ""], "4609": ["", "", "", ""], "4610": ["", "", "", ""], "4611": ["", "", "", ""], "4612": ["", "", "", ""], "4613": ["", "", "", ""], "4614": ["", "", "", ""], "4615": ["", "", "", ""], "4616": ["", "", "", ""], "4617": ["", "", "", ""], "4618": ["", "", "", ""], "4619": ["", "", "", ""], "4620": ["", "", "", ""], "4621": ["", "", "", ""], "4622": ["", "", "", ""], "4623": ["", "", "", ""], "4624": ["", "", "", ""], "4625": ["", "", "", ""], "4626": ["", "", "", ""], "4627": ["", "", "", ""], "4628": ["", "", "", ""], "4629": ["", "", "", ""], "4630": ["", "", "", ""], "4631": ["", "", "", ""], "4632": ["", "", "", ""], "4633": ["", "", "", ""], "4634": ["", "", "", ""], "4635": ["", "", "", ""], "4636": ["", "", "", ""], "4637": ["", "", "", ""], "4638": ["", "", "", ""], "4639": ["", "", "", ""], "4640": ["", "", "", ""], "4641": ["", "", "", ""], "4642": ["", "", "", ""], "4643": ["", "", "", ""], "4644": ["", "", "", ""], "4645": ["", "", "", ""], "4646": ["", "", "", ""], "4647": ["", "", "", ""], "4648": ["", "", "", ""], "4649": ["", "", "", ""], "4650": ["", "", "", ""], "4651": ["", "", "", ""], "4652": ["", "", "", ""], "4653": ["", "", "", ""], "4654": ["", "", "", ""], "4655": ["", "", "", ""], "4656": ["", "", "", ""], "4657": ["", "", "", ""], "4658": ["", "", "", ""], "4659": ["", "", "", ""], "4660": ["", "", "", ""], "4661": ["", "", "", ""], "4662": ["", "", "", ""], "4663": ["", "", "", ""], "4664": ["", "", "", ""], "4665": ["", "", "", ""], "4666": ["", "", "", ""], "4667": ["", "", "", ""], "4668": ["", "", "", ""], "4669": ["", "", "", ""], "4670": ["", "", "", ""], "4671": ["", "", "", ""], "4672": ["", "", "", ""], "4673": ["", "", "", ""], "4674": ["", "", "", ""], "4675": ["", "", "", ""], "4676": ["", "", "", ""], "4677": ["", "", "", ""], "4678": ["", "", "", ""], "4679": ["", "", "", ""], "4680": ["", "", "", ""], "4681": ["", "", "", ""], "4682": ["", "", "", ""], "4683": ["", "", "", ""], "4684": ["", "", "", ""], "4685": ["", "", "", ""], "4686": ["", "", "", ""], "4687": ["", "", "", ""], "4688": ["", "", "", ""], "4689": ["", "", "", ""], "4690": ["", "", "", ""], "4691": ["", "", "", ""], "4692": ["", "", "", ""], "4693": ["", "", "", ""], "4694": ["", "", "", ""], "4695": ["", "", "", ""], "4696": ["", "", "", ""], "4697": ["", "", "", ""], "4698": ["", "", "", ""], "4699": ["", "", "", ""], "4700": ["", "", "", ""], "4701": ["", "", "", ""], "4702": ["", "", "", ""], "4703": ["", "", "", ""], "4704": ["", "", "", ""], "4705": ["", "", "", ""], "4706": ["", "", "", ""], "4707": ["", "", "", ""], "4708": ["", "", "", ""], "4709": ["", "", "", ""], "4710": ["", "", "", ""], "4711": ["", "", "", ""], "4712": ["", "", "", ""], "4713": ["", "", "", ""], "4714": ["", "", "", ""], "4715": ["", "", "", ""], "4716": ["", "", "", ""], "4717": ["", "", "", ""], "4718": ["", "", "", ""], "4719": ["", "", "", ""], "4720": ["", "", "", ""], "4721": ["", "", "", ""], "4722": ["", "", "", ""], "4723": ["", "", "", ""], "4724": ["", "", "", ""], "4725": ["", "", "", ""], "4726": ["", "", "", ""], "4727": ["", "", "", ""], "4728": ["", "", "", ""], "4729": ["", "", "", ""], "4730": ["", "", "", ""], "4731": ["", "", "", ""], "4732": ["", "", "", ""], "4733": ["", "", "", ""], "4734": ["", "", "", ""], "4735": ["", "", "", ""], "4736": ["", "", "", ""], "4737": ["", "", "", ""], "4738": ["", "", "", ""], "4739": ["", "", "", ""], "4740": ["", "", "", ""], "4741": ["", "", "", ""], "4742": ["", "", "", ""], "4743": ["", "", "", ""], "4744": ["", "", "", ""], "4745": ["", "", "", ""], "4746": ["", "", "", ""], "4747": ["", "", "", ""], "4748": ["", "", "", ""], "4749": ["", "", "", ""], "4750": ["", "", "", ""], "4751": ["", "", "", ""], "4752": ["", "", "", ""], "4753": ["", "", "", ""], "4754": ["", "", "", ""], "4755": ["", "", "", ""], "4756": ["", "", "", ""], "4757": ["", "", "", ""], "4758": ["", "", "", ""], "4759": ["", "", "", ""], "4760": ["", "", "", ""], "4761": ["", "", "", ""], "4762": ["", "", "", ""], "4763": ["", "", "", ""], "4764": ["", "", "", ""], "4765": ["", "", "", ""], "4766": ["", "", "", ""], "4767": ["", "", "", ""], "4768": ["", "", "", ""], "4769": ["", "", "", ""], "4770": ["", "", "", ""], "4771": ["", "", "", ""], "4772": ["", "", "", ""], "4773": ["", "", "", ""], "4774": ["", "", "", ""], "4775": ["", "", "", ""], "4776": ["", "", "", ""], "4777": ["", "", "", ""], "4778": ["", "", "", ""], "4779": ["", "", "", ""], "4780": ["", "", "", ""], "4781": ["", "", "", ""], "4782": ["", "", "", ""], "4783": ["", "", "", ""], "4784": ["", "", "", ""], "4785": ["", "", "", ""], "4786": ["", "", "", ""], "4787": ["", "", "", ""], "4788": ["", "", "", ""], "4789": ["", "", "", ""], "4790": ["", "", "", ""], "4791": ["", "", "", ""], "4792": ["", "", "", ""], "4793": ["", "", "", ""], "4794": ["", "", "", ""], "4795": ["", "", "", ""], "4796": ["", "", "", ""], "4797": ["", "", "", ""], "4798": ["", "", "", ""], "4799": ["", "", "", ""], "4800": ["", "", "", ""], "4801": ["", "", "", ""], "4802": ["", "", "", ""], "4803": ["", "", "", ""], "4804": ["", "", "", ""], "4805": ["", "", "", ""], "4806": ["", "", "", ""], "4807": ["", "", "", ""], "4808": ["", "", "", ""], "4809": ["", "", "", ""], "4810": ["", "", "", ""], "4811": ["", "", "", ""], "4812": ["", "", "", ""], "4813": ["", "", "", ""], "4814": ["", "", "", ""], "4815": ["", "", "", ""], "4816": ["", "", "", ""], "4817": ["", "", "", ""], "4818": ["", "", "", ""], "4819": ["", "", "", ""], "4820": ["", "", "", ""], "4821": ["", "", "", ""], "4822": ["", "", "", ""], "4823": ["", "", "", ""], "4824": ["", "", "", ""], "4825": ["", "", "", ""], "4826": ["", "", "", ""], "4827": ["", "", "", ""], "4828": ["", "", "", ""], "4829": ["", "", "", ""], "4830": ["", "", "", ""], "4831": ["", "", "", ""], "4832": ["", "", "", ""], "4833": ["", "", "", ""], "4834": ["", "", "", ""], "4835": ["", "", "", ""], "4836": ["", "", "", ""], "4837": ["", "", "", ""], "4838": ["", "", "", ""], "4839": ["", "", "", ""], "4840": ["", "", "", ""], "4841": ["", "", "", ""], "4842": ["", "", "", ""], "4843": ["", "", "", ""], "4844": ["", "", "", ""], "4845": ["", "", "", ""], "4846": ["", "", "", ""], "4847": ["", "", "", ""], "4848": ["", "", "", ""], "4849": ["", "", "", ""], "4850": ["", "", "", ""], "4851": ["", "", "", ""], "4852": ["", "", "", ""], "4853": ["", "", "", ""], "4854": ["", "", "", ""], "4855": ["", "", "", ""], "4856": ["", "", "", ""], "4857": ["", "", "", ""], "4858": ["", "", "", ""], "4859": ["", "", "", ""], "4860": ["", "", "", ""], "4861": ["", "", "", ""], "4862": ["", "", "", ""], "4863": ["", "", "", ""], "4864": ["", "", "", ""], "4865": ["", "", "", ""], "4866": ["", "", "", ""], "4867": ["", "", "", ""], "4868": ["", "", "", ""], "4869": ["", "", "", ""], "4870": ["", "", "", ""], "4871": ["", "", "", ""], "4872": ["", "", "", ""], "4873": ["", "", "", ""], "4874": ["", "", "", ""], "4875": ["", "", "", ""], "4876": ["", "", "", ""], "4877": ["", "", "", ""], "4878": ["", "", "", ""], "4879": ["", "", "", ""], "4880": ["", "", "", ""], "4881": ["", "", "", ""], "4882": ["", "", "", ""], "4883": ["", "", "", ""], "4884": ["", "", "", ""], "4885": ["", "", "", ""], "4886": ["", "", "", ""], "4887": ["", "", "", ""], "4888": ["", "", "", ""], "4889": ["", "", "", ""], "4890": ["", "", "", ""], "4891": ["", "", "", ""], "4892": ["", "", "", ""], "4893": ["", "", "", ""], "4894": ["", "", "", ""], "4895": ["", "", "", ""], "4896": ["", "", "", ""], "4897": ["", "", "", ""], "4898": ["", "", "", ""], "4899": ["", "", "", ""], "4900": ["", "", "", ""], "4901": ["", "", "", ""], "4902": ["", "", "", ""], "4903": ["", "", "", ""], "4904": ["", "", "", ""], "4905": ["", "", "", ""], "4906": ["", "", "", ""], "4907": ["", "", "", ""], "4908": ["", "", "", ""], "4909": ["", "", "", ""], "4910": ["", "", "", ""], "4911": ["", "", "", ""], "4912": ["", "", "", ""], "4913": ["", "", "", ""], "4914": ["", "", "", ""], "4915": ["", "", "", ""], "4916": ["", "", "", ""], "4917": ["", "", "", ""], "4918": ["", "", "", ""], "4919": ["", "", "", ""], "4920": ["", "", "", ""], "4921": ["", "", "", ""], "4922": ["", "", "", ""], "4923": ["", "", "", ""], "4924": ["", "", "", ""], "4925": ["", "", "", ""], "4926": ["", "", "", ""], "4927": ["", "", "", ""], "4928": ["", "", "", ""], "4929": ["", "", "", ""], "4930": ["", "", "", ""], "4931": ["", "", "", ""], "4932": ["", "", "", ""], "4933": ["", "", "", ""], "4934": ["", "", "", ""], "4935": ["", "", "", ""], "4936": ["", "", "", ""], "4937": ["", "", "", ""], "4938": ["", "", "", ""], "4939": ["", "", "", ""], "4940": ["", "", "", ""], "4941": ["", "", "", ""], "4942": ["", "", "", ""], "4943": ["", "", "", ""], "4944": ["", "", "", ""], "4945": ["", "", "", ""], "4946": ["", "", "", ""], "4947": ["", "", "", ""], "4948": ["", "", "", ""], "4949": ["", "", "", ""], "4950": ["", "", "", ""], "4951": ["", "", "", ""], "4952": ["", "", "", ""], "4953": ["", "", "", ""], "4954": ["", "", "", ""], "4955": ["", "", "", ""], "4956": ["", "", "", ""], "4957": ["", "", "", ""], "4958": ["", "", "", ""], "4959": ["", "", "", ""], "4960": ["", "", "", ""], "4961": ["", "", "", ""], "4962": ["", "", "", ""], "4963": ["", "", "", ""], "4964": ["", "", "", ""], "4965": ["", "", "", ""], "4966": ["", "", "", ""], "4967": ["", "", "", ""], "4968": ["", "", "", ""], "4969": ["", "", "", ""], "4970": ["", "", "", ""], "4971": ["", "", "", ""], "4972": ["", "", "", ""], "4973": ["", "", "", ""], "4974": ["", "", "", ""], "4975": ["", "", "", ""], "4976": ["", "", "", ""], "4977": ["", "", "", ""], "4978": ["", "", "", ""], "4979": ["", "", "", ""], "4980": ["", "", "", ""], "4981": ["", "", "", ""], "4982": ["", "", "", ""], "4983": ["", "", "", ""], "4984": ["", "", "", ""], "4985": ["", "", "", ""], "4986": ["", "", "", ""], "4987": ["", "", "", ""], "4988": ["", "", "", ""], "4989": ["", "", "", ""], "4990": ["", "", "", ""], "4991": ["", "", "", ""], "4992": ["", "", "", ""], "4993": ["", "", "", ""], "4994": ["", "", "", ""], "4995": ["", "", "", ""], "4996": ["", "", "", ""], "4997": ["", "", "", ""], "4998": ["", "", "", ""], "4999": ["", "", "", ""], "5000": ["", "", "", ""], "5001": ["", "", "", ""], "5002": ["", "", "", ""], "5003": ["", "", "", ""], "5004": ["", "", "", ""], "5005": ["", "", "", ""], "5006": ["", "", "", ""], "5007": ["", "", "", ""], "5008": ["", "", "", ""], "5009": ["", "", "", ""], "5010": ["", "", "", ""], "5011": ["", "", "", ""], "5012": ["", "", "", ""], "5013": ["", "", "", ""], "5014": ["", "", "", ""], "5015": ["", "", "", ""], "5016": ["", "", "", ""], "5017": ["", "", "", ""], "5018": ["", "", "", ""], "5019": ["", "", "", ""], "5020": ["", "", "", ""], "5021": ["", "", "", ""], "5022": ["", "", "", ""], "5023": ["", "", "", ""], "5024": ["", "", "", ""], "5025": ["", "", "", ""], "5026": ["", "", "", ""], "5027": ["", "", "", ""], "5028": ["", "", "", ""], "5029": ["", "", "", ""], "5030": ["", "", "", ""], "5031": ["", "", "", ""], "5032": ["", "", "", ""], "5033": ["", "", "", ""], "5034": ["", "", "", ""], "5035": ["", "", "", ""], "5036": ["", "", "", ""], "5037": ["", "", "", ""], "5038": ["", "", "", ""], "5039": ["", "", "", ""], "5040": ["", "", "", ""], "5041": ["", "", "", ""], "5042": ["", "", "", ""], "5043": ["", "", "", ""], "5044": ["", "", "", ""], "5045": ["", "", "", ""], "5046": ["", "", "", ""], "5047": ["", "", "", ""], "5048": ["", "", "", ""], "5049": ["", "", "", ""], "5050": ["", "", "", ""], "5051": ["", "", "", ""], "5052": ["", "", "", ""], "5053": ["", "", "", ""], "5054": ["", "", "", ""], "5055": ["", "", "", ""], "5056": ["", "", "", ""], "5057": ["", "", "", ""], "5058": ["", "", "", ""], "5059": ["", "", "", ""], "5060": ["", "", "", ""], "5061": ["", "", "", ""], "5062": ["", "", "", ""], "5063": ["", "", "", ""], "5064": ["", "", "", ""], "5065": ["", "", "", ""], "5066": ["", "", "", ""], "5067": ["", "", "", ""], "5068": ["", "", "", ""], "5069": ["", "", "", ""], "5070": ["", "", "", ""], "5071": ["", "", "", ""], "5072": ["", "", "", ""], "5073": ["", "", "", ""], "5074": ["", "", "", ""], "5075": ["", "", "", ""], "5076": ["", "", "", ""], "5077": ["", "", "", ""], "5078": ["", "", "", ""], "5079": ["", "", "", ""], "5080": ["", "", "", ""], "5081": ["", "", "", ""], "5082": ["", "", "", ""], "5083": ["", "", "", ""], "5084": ["", "", "", ""], "5085": ["", "", "", ""], "5086": ["", "", "", ""], "5087": ["", "", "", ""], "5088": ["", "", "", ""], "5089": ["", "", "", ""], "5090": ["", "", "", ""], "5091": ["", "", "", ""], "5092": ["", "", "", ""], "5093": ["", "", "", ""], "5094": ["", "", "", ""], "5095": ["", "", "", ""], "5096": ["", "", "", ""], "5097": ["", "", "", ""], "5098": ["", "", "", ""], "5099": ["", "", "", ""], "5100": ["", "", "", ""], "5101": ["", "", "", ""], "5102": ["", "", "", ""], "5103": ["", "", "", ""], "5104": ["", "", "", ""], "5105": ["", "", "", ""], "5106": ["", "", "", ""], "5107": ["", "", "", ""], "5108": ["", "", "", ""], "5109": ["", "", "", ""], "5110": ["", "", "", ""], "5111": ["", "", "", ""], "5112": ["", "", "", ""], "5113": ["", "", "", ""], "5114": ["", "", "", ""], "5115": ["", "", "", ""], "5116": ["", "", "", ""], "5117": ["", "", "", ""], "5118": ["", "", "", ""], "5119": ["", "", "", ""], "5120": ["", "", "", ""], "5121": ["", "", "", ""], "5122": ["", "", "", ""], "5123": ["", "", "", ""], "5124": ["", "", "", ""], "5125": ["", "", "", ""], "5126": ["", "", "", ""], "5127": ["", "", "", ""], "5128": ["", "", "", ""], "5129": ["", "", "", ""], "5130": ["", "", "", ""], "5131": ["", "", "", ""], "5132": ["", "", "", ""], "5133": ["", "", "", ""], "5134": ["", "", "", ""], "5135": ["", "", "", ""], "5136": ["", "", "", ""], "5137": ["", "", "", ""], "5138": ["", "", "", ""], "5139": ["", "", "", ""], "5140": ["", "", "", ""], "5141": ["", "", "", ""], "5142": ["", "", "", ""], "5143": ["", "", "", ""], "5144": ["", "", "", ""], "5145": ["", "", "", ""], "5146": ["", "", "", ""], "5147": ["", "", "", ""], "5148": ["", "", "", ""], "5149": ["", "", "", ""], "5150": ["", "", "", ""], "5151": ["", "", "", ""], "5152": ["", "", "", ""], "5153": ["", "", "", ""], "5154": ["", "", "", ""], "5155": ["", "", "", ""], "5156": ["", "", "", ""], "5157": ["", "", "", ""], "5158": ["", "", "", ""], "5159": ["", "", "", ""], "5160": ["", "", "", ""], "5161": ["", "", "", ""], "5162": ["", "", "", ""], "5163": ["", "", "", ""], "5164": ["", "", "", ""], "5165": ["", "", "", ""], "5166": ["", "", "", ""], "5167": ["", "", "", ""], "5168": ["", "", "", ""], "5169": ["", "", "", ""], "5170": ["", "", "", ""], "5171": ["", "", "", ""], "5172": ["", "", "", ""], "5173": ["", "", "", ""], "5174": ["", "", "", ""], "5175": ["", "", "", ""], "5176": ["", "", "", ""], "5177": ["", "", "", ""], "5178": ["", "", "", ""], "5179": ["", "", "", ""], "5180": ["", "", "", ""], "5181": ["", "", "", ""], "5182": ["", "", "", ""], "5183": ["", "", "", ""], "5184": ["", "", "", ""], "5185": ["", "", "", ""], "5186": ["", "", "", ""], "5187": ["", "", "", ""], "5188": ["", "", "", ""], "5189": ["", "", "", ""], "5190": ["", "", "", ""], "5191": ["", "", "", ""], "5192": ["", "", "", ""], "5193": ["", "", "", ""], "5194": ["", "", "", ""], "5195": ["", "", "", ""], "5196": ["", "", "", ""], "5197": ["", "", "", ""], "5198": ["", "", "", ""], "5199": ["", "", "", ""], "5200": ["", "", "", ""], "5201": ["", "", "", ""], "5202": ["", "", "", ""], "5203": ["", "", "", ""], "5204": ["", "", "", ""], "5205": ["", "", "", ""], "5206": ["", "", "", ""], "5207": ["", "", "", ""], "5208": ["", "", "", ""], "5209": ["", "", "", ""], "5210": ["", "", "", ""], "5211": ["", "", "", ""], "5212": ["", "", "", ""], "5213": ["", "", "", ""], "5214": ["", "", "", ""], "5215": ["", "", "", ""], "5216": ["", "", "", ""], "5217": ["", "", "", ""], "5218": ["", "", "", ""], "5219": ["", "", "", ""], "5220": ["", "", "", ""], "5221": ["", "", "", ""], "5222": ["", "", "", ""], "5223": ["", "", "", ""], "5224": ["", "", "", ""], "5225": ["", "", "", ""], "5226": ["", "", "", ""], "5227": ["", "", "", ""], "5228": ["", "", "", ""], "5229": ["", "", "", ""], "5230": ["", "", "", ""], "5231": ["", "", "", ""], "5232": ["", "", "", ""], "5233": ["", "", "", ""], "5234": ["", "", "", ""], "5235": ["", "", "", ""], "5236": ["", "", "", ""], "5237": ["", "", "", ""], "5238": ["", "", "", ""], "5239": ["", "", "", ""], "5240": ["", "", "", ""], "5241": ["", "", "", ""], "5242": ["", "", "", ""], "5243": ["", "", "", ""], "5244": ["", "", "", ""], "5245": ["", "", "", ""], "5246": ["", "", "", ""], "5247": ["", "", "", ""], "5248": ["", "", "", ""], "5249": ["", "", "", ""], "5250": ["", "", "", ""], "5251": ["", "", "", ""], "5252": ["", "", "", ""], "5253": ["", "", "", ""], "5254": ["", "", "", ""], "5255": ["", "", "", ""], "5256": ["", "", "", ""], "5257": ["", "", "", ""], "5258": ["", "", "", ""], "5259": ["", "", "", ""], "5260": ["", "", "", ""], "5261": ["", "", "", ""], "5262": ["", "", "", ""], "5263": ["", "", "", ""], "5264": ["", "", "", ""], "5265": ["", "", "", ""], "5266": ["", "", "", ""], "5267": ["", "", "", ""], "5268": ["", "", "", ""], "5269": ["", "", "", ""], "5270": ["", "", "", ""], "5271": ["", "", "", ""], "5272": ["", "", "", ""], "5273": ["", "", "", ""], "5274": ["", "", "", ""], "5275": ["", "", "", ""], "5276": ["", "", "", ""], "5277": ["", "", "", ""], "5278": ["", "", "", ""], "5279": ["", "", "", ""], "5280": ["", "", "", ""], "5281": ["", "", "", ""], "5282": ["", "", "", ""], "5283": ["", "", "", ""], "5284": ["", "", "", ""], "5285": ["", "", "", ""], "5286": ["", "", "", ""], "5287": ["", "", "", ""], "5288": ["", "", "", ""], "5289": ["", "", "", ""], "5290": ["", "", "", ""], "5291": ["", "", "", ""], "5292": ["", "", "", ""], "5293": ["", "", "", ""], "5294": ["", "", "", ""], "5295": ["", "", "", ""], "5296": ["", "", "", ""], "5297": ["", "", "", ""], "5298": ["", "", "", ""], "5299": ["", "", "", ""], "5300": ["", "", "", ""], "5301": ["", "", "", ""], "5302": ["", "", "", ""], "5303": ["", "", "", ""], "5304": ["", "", "", ""], "5305": ["", "", "", ""], "5306": ["", "", "", ""], "5307": ["", "", "", ""], "5308": ["", "", "", ""], "5309": ["", "", "", ""], "5310": ["", "", "", ""], "5311": ["", "", "", ""], "5312": ["", "", "", ""], "5313": ["", "", "", ""], "5314": ["", "", "", ""], "5315": ["", "", "", ""], "5316": ["", "", "", ""], "5317": ["", "", "", ""], "5318": ["", "", "", ""], "5319": ["", "", "", ""], "5320": ["", "", "", ""], "5321": ["", "", "", ""], "5322": ["", "", "", ""], "5323": ["", "", "", ""], "5324": ["", "", "", ""], "5325": ["", "", "", ""], "5326": ["", "", "", ""], "5327": ["", "", "", ""], "5328": ["", "", "", ""], "5329": ["", "", "", ""], "5330": ["", "", "", ""], "5331": ["", "", "", ""], "5332": ["", "", "", ""], "5333": ["", "", "", ""], "5334": ["", "", "", ""], "5335": ["", "", "", ""], "5336": ["", "", "", ""], "5337": ["", "", "", ""], "5338": ["", "", "", ""], "5339": ["", "", "", ""], "5340": ["", "", "", ""], "5341": ["", "", "", ""], "5342": ["", "", "", ""], "5343": ["", "", "", ""], "5344": ["", "", "", ""], "5345": ["", "", "", ""], "5346": ["", "", "", ""], "5347": ["", "", "", ""], "5348": ["", "", "", ""], "5349": ["", "", "", ""], "5350": ["", "", "", ""], "5351": ["", "", "", ""], "5352": ["", "", "", ""], "5353": ["", "", "", ""], "5354": ["", "", "", ""], "5355": ["", "", "", ""], "5356": ["", "", "", ""], "5357": ["", "", "", ""], "5358": ["", "", "", ""], "5359": ["", "", "", ""], "5360": ["", "", "", ""], "5361": ["", "", "", ""], "5362": ["", "", "", ""], "5363": ["", "", "", ""], "5364": ["", "", "", ""], "5365": ["", "", "", ""], "5366": ["", "", "", ""], "5367": ["", "", "", ""], "5368": ["", "", "", ""], "5369": ["", "", "", ""], "5370": ["", "", "", ""], "5371": ["", "", "", ""], "5372": ["", "", "", ""], "5373": ["", "", "", ""], "5374": ["", "", "", ""], "5375": ["", "", "", ""], "5376": ["", "", "", ""], "5377": ["", "", "", ""], "5378": ["", "", "", ""], "5379": ["", "", "", ""], "5380": ["", "", "", ""], "5381": ["", "", "", ""], "5382": ["", "", "", ""], "5383": ["", "", "", ""], "5384": ["", "", "", ""], "5385": ["", "", "", ""], "5386": ["", "", "", ""], "5387": ["", "", "", ""], "5388": ["", "", "", ""], "5389": ["", "", "", ""], "5390": ["", "", "", ""], "5391": ["", "", "", ""], "5392": ["", "", "", ""], "5393": ["", "", "", ""], "5394": ["", "", "", ""], "5395": ["", "", "", ""], "5396": ["", "", "", ""], "5397": ["", "", "", ""], "5398": ["", "", "", ""], "5399": ["", "", "", ""], "5400": ["", "", "", ""], "5401": ["", "", "", ""], "5402": ["", "", "", ""], "5403": ["", "", "", ""], "5404": ["", "", "", ""], "5405": ["", "", "", ""], "5406": ["", "", "", ""], "5407": ["", "", "", ""], "5408": ["", "", "", ""], "5409": ["", "", "", ""], "5410": ["", "", "", ""], "5411": ["", "", "", ""], "5412": ["", "", "", ""], "5413": ["", "", "", ""], "5414": ["", "", "", ""], "5415": ["", "", "", ""], "5416": ["", "", "", ""], "5417": ["", "", "", ""], "5418": ["", "", "", ""], "5419": ["", "", "", ""], "5420": ["", "", "", ""], "5421": ["", "", "", ""], "5422": ["", "", "", ""], "5423": ["", "", "", ""], "5424": ["", "", "", ""], "5425": ["", "", "", ""], "5426": ["", "", "", ""], "5427": ["", "", "", ""], "5428": ["", "", "", ""], "5429": ["", "", "", ""], "5430": ["", "", "", ""], "5431": ["", "", "", ""], "5432": ["", "", "", ""], "5433": ["", "", "", ""], "5434": ["", "", "", ""], "5435": ["", "", "", ""], "5436": ["", "", "", ""], "5437": ["", "", "", ""], "5438": ["", "", "", ""], "5439": ["", "", "", ""], "5440": ["", "", "", ""], "5441": ["", "", "", ""], "5442": ["", "", "", ""], "5443": ["", "", "", ""], "5444": ["", "", "", ""], "5445": ["", "", "", ""], "5446": ["", "", "", ""], "5447": ["", "", "", ""], "5448": ["", "", "", ""], "5449": ["", "", "", ""], "5450": ["", "", "", ""], "5451": ["", "", "", ""], "5452": ["", "", "", ""], "5453": ["", "", "", ""], "5454": ["", "", "", ""], "5455": ["", "", "", ""], "5456": ["", "", "", ""], "5457": ["", "", "", ""], "5458": ["", "", "", ""], "5459": ["", "", "", ""], "5460": ["", "", "", ""], "5461": ["", "", "", ""], "5462": ["", "", "", ""], "5463": ["", "", "", ""], "5464": ["", "", "", ""], "5465": ["", "", "", ""], "5466": ["", "", "", ""], "5467": ["", "", "", ""], "5468": ["", "", "", ""], "5469": ["", "", "", ""], "5470": ["", "", "", ""], "5471": ["", "", "", ""], "5472": ["", "", "", ""], "5473": ["", "", "", ""], "5474": ["", "", "", ""], "5475": ["", "", "", ""], "5476": ["", "", "", ""], "5477": ["", "", "", ""], "5478": ["", "", "", ""], "5479": ["", "", "", ""], "5480": ["", "", "", ""], "5481": ["", "", "", ""], "5482": ["", "", "", ""], "5483": ["", "", "", ""], "5484": ["", "", "", ""], "5485": ["", "", "", ""], "5486": ["", "", "", ""], "5487": ["", "", "", ""], "5488": ["", "", "", ""], "5489": ["", "", "", ""], "5490": ["", "", "", ""], "5491": ["", "", "", ""], "5492": ["", "", "", ""], "5493": ["", "", "", ""], "5494": ["", "", "", ""], "5495": ["", "", "", ""], "5496": ["", "", "", ""], "5497": ["", "", "", ""], "5498": ["", "", "", ""], "5499": ["", "", "", ""], "5500": ["", "", "", ""], "5501": ["", "", "", ""], "5502": ["", "", "", ""], "5503": ["", "", "", ""], "5504": ["", "", "", ""], "5505": ["", "", "", ""], "5506": ["", "", "", ""], "5507": ["", "", "", ""], "5508": ["", "", "", ""], "5509": ["", "", "", ""], "5510": ["", "", "", ""], "5511": ["", "", "", ""], "5512": ["", "", "", ""], "5513": ["", "", "", ""], "5514": ["", "", "", ""], "5515": ["", "", "", ""], "5516": ["", "", "", ""], "5517": ["", "", "", ""], "5518": ["", "", "", ""], "5519": ["", "", "", ""], "5520": ["", "", "", ""], "5521": ["", "", "", ""], "5522": ["", "", "", ""], "5523": ["", "", "", ""], "5524": ["", "", "", ""], "5525": ["", "", "", ""], "5526": ["", "", "", ""], "5527": ["", "", "", ""], "5528": ["", "", "", ""], "5529": ["", "", "", ""], "5530": ["", "", "", ""], "5531": ["", "", "", ""], "5532": ["", "", "", ""], "5533": ["", "", "", ""], "5534": ["", "", "", ""], "5535": ["", "", "", ""], "5536": ["", "", "", ""], "5537": ["", "", "", ""], "5538": ["", "", "", ""], "5539": ["", "", "", ""], "5540": ["", "", "", ""], "5541": ["", "", "", ""], "5542": ["", "", "", ""], "5543": ["", "", "", ""], "5544": ["", "", "", ""], "5545": ["", "", "", ""], "5546": ["", "", "", ""], "5547": ["", "", "", ""], "5548": ["", "", "", ""], "5549": ["", "", "", ""], "5550": ["", "", "", ""], "5551": ["", "", "", ""], "5552": ["", "", "", ""], "5553": ["", "", "", ""], "5554": ["", "", "", ""], "5555": ["", "", "", ""], "5556": ["", "", "", ""], "5557": ["", "", "", ""], "5558": ["", "", "", ""], "5559": ["", "", "", ""], "5560": ["", "", "", ""], "5561": ["", "", "", ""], "5562": ["", "", "", ""], "5563": ["", "", "", ""], "5564": ["", "", "", ""], "5565": ["", "", "", ""], "5566": ["", "", "", ""], "5567": ["", "", "", ""], "5568": ["", "", "", ""], "5569": ["", "", "", ""], "5570": ["", "", "", ""], "5571": ["", "", "", ""], "5572": ["", "", "", ""], "5573": ["", "", "", ""], "5574": ["", "", "", ""], "5575": ["", "", "", ""], "5576": ["", "", "", ""], "5577": ["", "", "", ""], "5578": ["", "", "", ""], "5579": ["", "", "", ""], "5580": ["", "", "", ""], "5581": ["", "", "", ""], "5582": ["", "", "", ""], "5583": ["", "", "", ""], "5584": ["", "", "", ""], "5585": ["", "", "", ""], "5586": ["", "", "", ""], "5587": ["", "", "", ""], "5588": ["", "", "", ""], "5589": ["", "", "", ""], "5590": ["", "", "", ""], "5591": ["", "", "", ""], "5592": ["", "", "", ""], "5593": ["", "", "", ""], "5594": ["", "", "", ""], "5595": ["", "", "", ""], "5596": ["", "", "", ""], "5597": ["", "", "", ""], "5598": ["", "", "", ""], "5599": ["", "", "", ""], "5600": ["", "", "", ""], "5601": ["", "", "", ""], "5602": ["", "", "", ""], "5603": ["", "", "", ""], "5604": ["", "", "", ""], "5605": ["", "", "", ""], "5606": ["", "", "", ""], "5607": ["", "", "", ""], "5608": ["", "", "", ""], "5609": ["", "", "", ""], "5610": ["", "", "", ""], "5611": ["", "", "", ""], "5612": ["", "", "", ""], "5613": ["", "", "", ""], "5614": ["", "", "", ""], "5615": ["", "", "", ""], "5616": ["", "", "", ""], "5617": ["", "", "", ""], "5618": ["", "", "", ""], "5619": ["", "", "", ""], "5620": ["", "", "", ""], "5621": ["", "", "", ""], "5622": ["", "", "", ""], "5623": ["", "", "", ""], "5624": ["", "", "", ""], "5625": ["", "", "", ""], "5626": ["", "", "", ""], "5627": ["", "", "", ""], "5628": ["", "", "", ""], "5629": ["", "", "", ""], "5630": ["", "", "", ""], "5631": ["", "", "", ""], "5632": ["", "", "", ""], "5633": ["", "", "", ""], "5634": ["", "", "", ""], "5635": ["", "", "", ""], "5636": ["", "", "", ""], "5637": ["", "", "", ""], "5638": ["", "", "", ""], "5639": ["", "", "", ""], "5640": ["", "", "", ""], "5641": ["", "", "", ""], "5642": ["", "", "", ""], "5643": ["", "", "", ""], "5644": ["", "", "", ""], "5645": ["", "", "", ""], "5646": ["", "", "", ""], "5647": ["", "", "", ""], "5648": ["", "", "", ""], "5649": ["", "", "", ""], "5650": ["", "", "", ""], "5651": ["", "", "", ""], "5652": ["", "", "", ""], "5653": ["", "", "", ""], "5654": ["", "", "", ""], "5655": ["", "", "", ""], "5656": ["", "", "", ""], "5657": ["", "", "", ""], "5658": ["", "", "", ""], "5659": ["", "", "", ""], "5660": ["", "", "", ""], "5661": ["", "", "", ""], "5662": ["", "", "", ""], "5663": ["", "", "", ""], "5664": ["", "", "", ""], "5665": ["", "", "", ""], "5666": ["", "", "", ""], "5667": ["", "", "", ""], "5668": ["", "", "", ""], "5669": ["", "", "", ""], "5670": ["", "", "", ""], "5671": ["", "", "", ""], "5672": ["", "", "", ""], "5673": ["", "", "", ""], "5674": ["", "", "", ""], "5675": ["", "", "", ""], "5676": ["", "", "", ""], "5677": ["", "", "", ""], "5678": ["", "", "", ""], "5679": ["", "", "", ""], "5680": ["", "", "", ""], "5681": ["", "", "", ""], "5682": ["", "", "", ""], "5683": ["", "", "", ""], "5684": ["", "", "", ""], "5685": ["", "", "", ""], "5686": ["", "", "", ""], "5687": ["", "", "", ""], "5688": ["", "", "", ""], "5689": ["", "", "", ""], "5690": ["", "", "", ""], "5691": ["", "", "", ""], "5692": ["", "", "", ""], "5693": ["", "", "", ""], "5694": ["", "", "", ""], "5695": ["", "", "", ""], "5696": ["", "", "", ""], "5697": ["", "", "", ""], "5698": ["", "", "", ""], "5699": ["", "", "", ""], "5700": ["", "", "", ""], "5701": ["", "", "", ""], "5702": ["", "", "", ""], "5703": ["", "", "", ""], "5704": ["", "", "", ""], "5705": ["", "", "", ""], "5706": ["", "", "", ""], "5707": ["", "", "", ""], "5708": ["", "", "", ""], "5709": ["", "", "", ""], "5710": ["", "", "", ""], "5711": ["", "", "", ""], "5712": ["", "", "", ""], "5713": ["", "", "", ""], "5714": ["", "", "", ""], "5715": ["", "", "", ""], "5716": ["", "", "", ""], "5717": ["", "", "", ""], "5718": ["", "", "", ""], "5719": ["", "", "", ""], "5720": ["", "", "", ""], "5721": ["", "", "", ""], "5722": ["", "", "", ""], "5723": ["", "", "", ""], "5724": ["", "", "", ""], "5725": ["", "", "", ""], "5726": ["", "", "", ""], "5727": ["", "", "", ""], "5728": ["", "", "", ""], "5729": ["", "", "", ""], "5730": ["", "", "", ""], "5731": ["", "", "", ""], "5732": ["", "", "", ""], "5733": ["", "", "", ""], "5734": ["", "", "", ""], "5735": ["", "", "", ""], "5736": ["", "", "", ""], "5737": ["", "", "", ""], "5738": ["", "", "", ""], "5739": ["", "", "", ""], "5740": ["", "", "", ""], "5741": ["", "", "", ""], "5742": ["", "", "", ""], "5743": ["", "", "", ""], "5744": ["", "", "", ""], "5745": ["", "", "", ""], "5746": ["", "", "", ""], "5747": ["", "", "", ""], "5748": ["", "", "", ""], "5749": ["", "", "", ""], "5750": ["", "", "", ""], "5751": ["", "", "", ""], "5752": ["", "", "", ""], "5753": ["", "", "", ""], "5754": ["", "", "", ""], "5755": ["", "", "", ""], "5756": ["", "", "", ""], "5757": ["", "", "", ""], "5758": ["", "", "", ""], "5759": ["", "", "", ""], "5760": ["", "", "", ""], "5761": ["", "", "", ""], "5762": ["", "", "", ""], "5763": ["", "", "", ""], "5764": ["", "", "", ""], "5765": ["", "", "", ""], "5766": ["", "", "", ""], "5767": ["", "", "", ""], "5768": ["", "", "", ""], "5769": ["", "", "", ""], "5770": ["", "", "", ""], "5771": ["", "", "", ""], "5772": ["", "", "", ""], "5773": ["", "", "", ""], "5774": ["", "", "", ""], "5775": ["", "", "", ""], "5776": ["", "", "", ""], "5777": ["", "", "", ""], "5778": ["", "", "", ""], "5779": ["", "", "", ""], "5780": ["", "", "", ""], "5781": ["", "", "", ""], "5782": ["", "", "", ""], "5783": ["", "", "", ""], "5784": ["", "", "", ""], "5785": ["", "", "", ""], "5786": ["", "", "", ""], "5787": ["", "", "", ""], "5788": ["", "", "", ""], "5789": ["", "", "", ""], "5790": ["", "", "", ""], "5791": ["", "", "", ""], "5792": ["", "", "", ""], "5793": ["", "", "", ""], "5794": ["", "", "", ""], "5795": ["", "", "", ""], "5796": ["", "", "", ""], "5797": ["", "", "", ""], "5798": ["", "", "", ""], "5799": ["", "", "", ""], "5800": ["", "", "", ""], "5801": ["", "", "", ""], "5802": ["", "", "", ""], "5803": ["", "", "", ""], "5804": ["", "", "", ""], "5805": ["", "", "", ""], "5806": ["", "", "", ""], "5807": ["", "", "", ""], "5808": ["", "", "", ""], "5809": ["", "", "", ""], "5810": ["", "", "", ""], "5811": ["", "", "", ""], "5812": ["", "", "", ""], "5813": ["", "", "", ""], "5814": ["", "", "", ""], "5815": ["", "", "", ""], "5816": ["", "", "", ""], "5817": ["", "", "", ""], "5818": ["", "", "", ""], "5819": ["", "", "", ""], "5820": ["", "", "", ""], "5821": ["", "", "", ""], "5822": ["", "", "", ""], "5823": ["", "", "", ""], "5824": ["", "", "", ""], "5825": ["", "", "", ""], "5826": ["", "", "", ""], "5827": ["", "", "", ""], "5828": ["", "", "", ""], "5829": ["", "", "", ""], "5830": ["", "", "", ""], "5831": ["", "", "", ""], "5832": ["", "", "", ""], "5833": ["", "", "", ""], "5834": ["", "", "", ""], "5835": ["", "", "", ""], "5836": ["", "", "", ""], "5837": ["", "", "", ""], "5838": ["", "", "", ""], "5839": ["", "", "", ""], "5840": ["", "", "", ""], "5841": ["", "", "", ""], "5842": ["", "", "", ""], "5843": ["", "", "", ""], "5844": ["", "", "", ""], "5845": ["", "", "", ""], "5846": ["", "", "", ""], "5847": ["", "", "", ""], "5848": ["", "", "", ""], "5849": ["", "", "", ""], "5850": ["", "", "", ""], "5851": ["", "", "", ""], "5852": ["", "", "", ""], "5853": ["", "", "", ""], "5854": ["", "", "", ""], "5855": ["", "", "", ""], "5856": ["", "", "", ""], "5857": ["", "", "", ""], "5858": ["", "", "", ""], "5859": ["", "", "", ""], "5860": ["", "", "", ""], "5861": ["", "", "", ""], "5862": ["", "", "", ""], "5863": ["", "", "", ""], "5864": ["", "", "", ""], "5865": ["", "", "", ""], "5866": ["", "", "", ""], "5867": ["", "", "", ""], "5868": ["", "", "", ""], "5869": ["", "", "", ""], "5870": ["", "", "", ""], "5871": ["", "", "", ""], "5872": ["", "", "", ""], "5873": ["", "", "", ""], "5874": ["", "", "", ""], "5875": ["", "", "", ""], "5876": ["", "", "", ""], "5877": ["", "", "", ""], "5878": ["", "", "", ""], "5879": ["", "", "", ""], "5880": ["", "", "", ""], "5881": ["", "", "", ""], "5882": ["", "", "", ""], "5883": ["", "", "", ""], "5884": ["", "", "", ""], "5885": ["", "", "", ""], "5886": ["", "", "", ""], "5887": ["", "", "", ""], "5888": ["", "", "", ""], "5889": ["", "", "", ""], "5890": ["", "", "", ""], "5891": ["", "", "", ""], "5892": ["", "", "", ""], "5893": ["", "", "", ""], "5894": ["", "", "", ""], "5895": ["", "", "", ""], "5896": ["", "", "", ""], "5897": ["", "", "", ""], "5898": ["", "", "", ""], "5899": ["", "", "", ""], "5900": ["", "", "", ""], "5901": ["", "", "", ""], "5902": ["", "", "", ""], "5903": ["", "", "", ""], "5904": ["", "", "", ""], "5905": ["", "", "", ""], "5906": ["", "", "", ""], "5907": ["", "", "", ""], "5908": ["", "", "", ""], "5909": ["", "", "", ""], "5910": ["", "", "", ""], "5911": ["", "", "", ""], "5912": ["", "", "", ""], "5913": ["", "", "", ""], "5914": ["", "", "", ""], "5915": ["", "", "", ""], "5916": ["", "", "", ""], "5917": ["", "", "", ""], "5918": ["", "", "", ""], "5919": ["", "", "", ""], "5920": ["", "", "", ""], "5921": ["", "", "", ""], "5922": ["", "", "", ""], "5923": ["", "", "", ""], "5924": ["", "", "", ""], "5925": ["", "", "", ""], "5926": ["", "", "", ""], "5927": ["", "", "", ""], "5928": ["", "", "", ""], "5929": ["", "", "", ""], "5930": ["", "", "", ""], "5931": ["", "", "", ""], "5932": ["", "", "", ""], "5933": ["", "", "", ""], "5934": ["", "", "", ""], "5935": ["", "", "", ""], "5936": ["", "", "", ""], "5937": ["", "", "", ""], "5938": ["", "", "", ""], "5939": ["", "", "", ""], "5940": ["", "", "", ""], "5941": ["", "", "", ""], "5942": ["", "", "", ""], "5943": ["", "", "", ""], "5944": ["", "", "", ""], "5945": ["", "", "", ""], "5946": ["", "", "", ""], "5947": ["", "", "", ""], "5948": ["", "", "", ""], "5949": ["", "", "", ""], "5950": ["", "", "", ""], "5951": ["", "", "", ""], "5952": ["", "", "", ""], "5953": ["", "", "", ""], "5954": ["", "", "", ""], "5955": ["", "", "", ""], "5956": ["", "", "", ""], "5957": ["", "", "", ""], "5958": ["", "", "", ""], "5959": ["", "", "", ""], "5960": ["", "", "", ""], "5961": ["", "", "", ""], "5962": ["", "", "", ""], "5963": ["", "", "", ""], "5964": ["", "", "", ""], "5965": ["", "", "", ""], "5966": ["", "", "", ""], "5967": ["", "", "", ""], "5968": ["", "", "", ""], "5969": ["", "", "", ""], "5970": ["", "", "", ""], "5971": ["", "", "", ""], "5972": ["", "", "", ""], "5973": ["", "", "", ""], "5974": ["", "", "", ""], "5975": ["", "", "", ""], "5976": ["", "", "", ""], "5977": ["", "", "", ""], "5978": ["", "", "", ""], "5979": ["", "", "", ""], "5980": ["", "", "", ""], "5981": ["", "", "", ""], "5982": ["", "", "", ""], "5983": ["", "", "", ""], "5984": ["", "", "", ""], "5985": ["", "", "", ""], "5986": ["", "", "", ""], "5987": ["", "", "", ""], "5988": ["", "", "", ""], "5989": ["", "", "", ""], "5990": ["", "", "", ""], "5991": ["", "", "", ""], "5992": ["", "", "", ""], "5993": ["", "", "", ""], "5994": ["", "", "", ""], "5995": ["", "", "", ""], "5996": ["", "", "", ""], "5997": ["", "", "", ""], "5998": ["", "", "", ""], "5999": ["", "", "", ""], "6000": ["", "", "", ""], "6001": ["", "", "", ""], "6002": ["", "", "", ""], "6003": ["", "", "", ""], "6004": ["", "", "", ""], "6005": ["", "", "", ""], "6006": ["", "", "", ""], "6007": ["", "", "", ""], "6008": ["", "", "", ""], "6009": ["", "", "", ""], "6010": ["", "", "", ""], "6011": ["", "", "", ""], "6012": ["", "", "", ""], "6013": ["", "", "", ""], "6014": ["", "", "", ""], "6015": ["", "", "", ""], "6016": ["", "", "", ""], "6017": ["", "", "", ""], "6018": ["", "", "", ""], "6019": ["", "", "", ""], "6020": ["", "", "", ""], "6021": ["", "", "", ""], "6022": ["", "", "", ""], "6023": ["", "", "", ""], "6024": ["", "", "", ""], "6025": ["", "", "", ""], "6026": ["", "", "", ""], "6027": ["", "", "", ""], "6028": ["", "", "", ""], "6029": ["", "", "", ""], "6030": ["", "", "", ""], "6031": ["", "", "", ""], "6032": ["", "", "", ""], "6033": ["", "", "", ""], "6034": ["", "", "", ""], "6035": ["", "", "", ""], "6036": ["", "", "", ""], "6037": ["", "", "", ""], "6038": ["", "", "", ""], "6039": ["", "", "", ""], "6040": ["", "", "", ""], "6041": ["", "", "", ""], "6042": ["", "", "", ""], "6043": ["", "", "", ""], "6044": ["", "", "", ""], "6045": ["", "", "", ""], "6046": ["", "", "", ""], "6047": ["", "", "", ""], "6048": ["", "", "", ""], "6049": ["", "", "", ""], "6050": ["", "", "", ""], "6051": ["", "", "", ""], "6052": ["", "", "", ""], "6053": ["", "", "", ""], "6054": ["", "", "", ""], "6055": ["", "", "", ""], "6056": ["", "", "", ""], "6057": ["", "", "", ""], "6058": ["", "", "", ""], "6059": ["", "", "", ""], "6060": ["", "", "", ""], "6061": ["", "", "", ""], "6062": ["", "", "", ""], "6063": ["", "", "", ""], "6064": ["", "", "", ""], "6065": ["", "", "", ""], "6066": ["", "", "", ""], "6067": ["", "", "", ""], "6068": ["", "", "", ""], "6069": ["", "", "", ""], "6070": ["", "", "", ""], "6071": ["", "", "", ""], "6072": ["", "", "", ""], "6073": ["", "", "", ""], "6074": ["", "", "", ""], "6075": ["", "", "", ""], "6076": ["", "", "", ""], "6077": ["", "", "", ""], "6078": ["", "", "", ""], "6079": ["", "", "", ""], "6080": ["", "", "", ""], "6081": ["", "", "", ""], "6082": ["", "", "", ""], "6083": ["", "", "", ""], "6084": ["", "", "", ""], "6085": ["", "", "", ""], "6086": ["", "", "", ""], "6087": ["", "", "", ""], "6088": ["", "", "", ""], "6089": ["", "", "", ""], "6090": ["", "", "", ""], "6091": ["", "", "", ""], "6092": ["", "", "", ""], "6093": ["", "", "", ""], "6094": ["", "", "", ""], "6095": ["", "", "", ""], "6096": ["", "", "", ""], "6097": ["", "", "", ""], "6098": ["", "", "", ""], "6099": ["", "", "", ""], "6100": ["", "", "", ""], "6101": ["", "", "", ""], "6102": ["", "", "", ""], "6103": ["", "", "", ""], "6104": ["", "", "", ""], "6105": ["", "", "", ""], "6106": ["", "", "", ""], "6107": ["", "", "", ""], "6108": ["", "", "", ""], "6109": ["", "", "", ""], "6110": ["", "", "", ""], "6111": ["", "", "", ""], "6112": ["", "", "", ""], "6113": ["", "", "", ""], "6114": ["", "", "", ""], "6115": ["", "", "", ""], "6116": ["", "", "", ""], "6117": ["", "", "", ""], "6118": ["", "", "", ""], "6119": ["", "", "", ""], "6120": ["", "", "", ""], "6121": ["", "", "", ""], "6122": ["", "", "", ""], "6123": ["", "", "", ""], "6124": ["", "", "", ""], "6125": ["", "", "", ""], "6126": ["", "", "", ""], "6127": ["", "", "", ""], "6128": ["", "", "", ""], "6129": ["", "", "", ""], "6130": ["", "", "", ""], "6131": ["", "", "", ""], "6132": ["", "", "", ""], "6133": ["", "", "", ""], "6134": ["", "", "", ""], "6135": ["", "", "", ""], "6136": ["", "", "", ""], "6137": ["", "", "", ""], "6138": ["", "", "", ""], "6139": ["", "", "", ""], "6140": ["", "", "", ""], "6141": ["", "", "", ""], "6142": ["", "", "", ""], "6143": ["", "", "", ""], "6144": ["", "", "", ""], "6145": ["", "", "", ""], "6146": ["", "", "", ""], "6147": ["", "", "", ""], "6148": ["", "", "", ""], "6149": ["", "", "", ""], "6150": ["", "", "", ""], "6151": ["", "", "", ""], "6152": ["", "", "", ""], "6153": ["", "", "", ""], "6154": ["", "", "", ""], "6155": ["", "", "", ""], "6156": ["", "", "", ""], "6157": ["", "", "", ""], "6158": ["", "", "", ""], "6159": ["", "", "", ""], "6160": ["", "", "", ""], "6161": ["", "", "", ""], "6162": ["", "", "", ""], "6163": ["", "", "", ""], "6164": ["", "", "", ""], "6165": ["", "", "", ""], "6166": ["", "", "", ""], "6167": ["", "", "", ""], "6168": ["", "", "", ""], "6169": ["", "", "", ""], "6170": ["", "", "", ""], "6171": ["", "", "", ""], "6172": ["", "", "", ""], "6173": ["", "", "", ""], "6174": ["", "", "", ""], "6175": ["", "", "", ""], "6176": ["", "", "", ""], "6177": ["", "", "", ""], "6178": ["", "", "", ""], "6179": ["", "", "", ""], "6180": ["", "", "", ""], "6181": ["", "", "", ""], "6182": ["", "", "", ""], "6183": ["", "", "", ""], "6184": ["", "", "", ""], "6185": ["", "", "", ""], "6186": ["", "", "", ""], "6187": ["", "", "", ""], "6188": ["", "", "", ""], "6189": ["", "", "", ""], "6190": ["", "", "", ""], "6191": ["", "", "", ""], "6192": ["", "", "", ""], "6193": ["", "", "", ""], "6194": ["", "", "", ""], "6195": ["", "", "", ""], "6196": ["", "", "", ""], "6197": ["", "", "", ""], "6198": ["", "", "", ""], "6199": ["", "", "", ""], "6200": ["", "", "", ""], "6201": ["", "", "", ""], "6202": ["", "", "", ""], "6203": ["", "", "", ""], "6204": ["", "", "", ""], "6205": ["", "", "", ""], "6206": ["", "", "", ""], "6207": ["", "", "", ""], "6208": ["", "", "", ""], "6209": ["", "", "", ""], "6210": ["", "", "", ""], "6211": ["", "", "", ""], "6212": ["", "", "", ""], "6213": ["", "", "", ""], "6214": ["", "", "", ""], "6215": ["", "", "", ""], "6216": ["", "", "", ""], "6217": ["", "", "", ""], "6218": ["", "", "", ""], "6219": ["", "", "", ""], "6220": ["", "", "", ""], "6221": ["", "", "", ""], "6222": ["", "", "", ""], "6223": ["", "", "", ""], "6224": ["", "", "", ""], "6225": ["", "", "", ""], "6226": ["", "", "", ""], "6227": ["", "", "", ""], "6228": ["", "", "", ""], "6229": ["", "", "", ""], "6230": ["", "", "", ""], "6231": ["", "", "", ""], "6232": ["", "", "", ""], "6233": ["", "", "", ""], "6234": ["", "", "", ""], "6235": ["", "", "", ""], "6236": ["", "", "", ""], "6237": ["", "", "", ""], "6238": ["", "", "", ""], "6239": ["", "", "", ""], "6240": ["", "", "", ""], "6241": ["", "", "", ""], "6242": ["", "", "", ""], "6243": ["", "", "", ""], "6244": ["", "", "", ""], "6245": ["", "", "", ""], "6246": ["", "", "", ""], "6247": ["", "", "", ""], "6248": ["", "", "", ""], "6249": ["", "", "", ""], "6250": ["", "", "", ""], "6251": ["", "", "", ""], "6252": ["", "", "", ""], "6253": ["", "", "", ""], "6254": ["", "", "", ""], "6255": ["", "", "", ""], "6256": ["", "", "", ""], "6257": ["", "", "", ""], "6258": ["", "", "", ""], "6259": ["", "", "", ""], "6260": ["", "", "", ""], "6261": ["", "", "", ""], "6262": ["", "", "", ""], "6263": ["", "", "", ""], "6264": ["", "", "", ""], "6265": ["", "", "", ""], "6266": ["", "", "", ""], "6267": ["", "", "", ""], "6268": ["", "", "", ""], "6269": ["", "", "", ""], "6270": ["", "", "", ""], "6271": ["", "", "", ""], "6272": ["", "", "", ""], "6273": ["", "", "", ""], "6274": ["", "", "", ""], "6275": ["", "", "", ""], "6276": ["", "", "", ""], "6277": ["", "", "", ""], "6278": ["", "", "", ""], "6279": ["", "", "", ""], "6280": ["", "", "", ""], "6281": ["", "", "", ""], "6282": ["", "", "", ""], "6283": ["", "", "", ""], "6284": ["", "", "", ""], "6285": ["", "", "", ""], "6286": ["", "", "", ""], "6287": ["", "", "", ""], "6288": ["", "", "", ""], "6289": ["", "", "", ""], "6290": ["", "", "", ""], "6291": ["", "", "", ""], "6292": ["", "", "", ""], "6293": ["", "", "", ""], "6294": ["", "", "", ""], "6295": ["", "", "", ""], "6296": ["", "", "", ""], "6297": ["", "", "", ""], "6298": ["", "", "", ""], "6299": ["", "", "", ""], "6300": ["", "", "", ""], "6301": ["", "", "", ""], "6302": ["", "", "", ""], "6303": ["", "", "", ""], "6304": ["", "", "", ""], "6305": ["", "", "", ""], "6306": ["", "", "", ""], "6307": ["", "", "", ""], "6308": ["", "", "", ""], "6309": ["", "", "", ""], "6310": ["", "", "", ""], "6311": ["", "", "", ""], "6312": ["", "", "", ""], "6313": ["", "", "", ""], "6314": ["", "", "", ""], "6315": ["", "", "", ""], "6316": ["", "", "", ""], "6317": ["", "", "", ""], "6318": ["", "", "", ""], "6319": ["", "", "", ""], "6320": ["", "", "", ""], "6321": ["", "", "", ""], "6322": ["", "", "", ""], "6323": ["", "", "", ""], "6324": ["", "", "", ""], "6325": ["", "", "", ""], "6326": ["", "", "", ""], "6327": ["", "", "", ""], "6328": ["", "", "", ""], "6329": ["", "", "", ""], "6330": ["", "", "", ""], "6331": ["", "", "", ""], "6332": ["", "", "", ""], "6333": ["", "", "", ""], "6334": ["", "", "", ""], "6335": ["", "", "", ""], "6336": ["", "", "", ""], "6337": ["", "", "", ""], "6338": ["", "", "", ""], "6339": ["", "", "", ""], "6340": ["", "", "", ""], "6341": ["", "", "", ""], "6342": ["", "", "", ""], "6343": ["", "", "", ""], "6344": ["", "", "", ""], "6345": ["", "", "", ""], "6346": ["", "", "", ""], "6347": ["", "", "", ""], "6348": ["", "", "", ""], "6349": ["", "", "", ""], "6350": ["", "", "", ""], "6351": ["", "", "", ""], "6352": ["", "", "", ""], "6353": ["", "", "", ""], "6354": ["", "", "", ""], "6355": ["", "", "", ""], "6356": ["", "", "", ""], "6357": ["", "", "", ""], "6358": ["", "", "", ""], "6359": ["", "", "", ""], "6360": ["", "", "", ""], "6361": ["", "", "", ""], "6362": ["", "", "", ""], "6363": ["", "", "", ""], "6364": ["", "", "", ""], "6365": ["", "", "", ""], "6366": ["", "", "", ""], "6367": ["", "", "", ""], "6368": ["", "", "", ""], "6369": ["", "", "", ""], "6370": ["", "", "", ""], "6371": ["", "", "", ""], "6372": ["", "", "", ""], "6373": ["", "", "", ""], "6374": ["", "", "", ""], "6375": ["", "", "", ""], "6376": ["", "", "", ""], "6377": ["", "", "", ""], "6378": ["", "", "", ""], "6379": ["", "", "", ""], "6380": ["", "", "", ""], "6381": ["", "", "", ""], "6382": ["", "", "", ""], "6383": ["", "", "", ""], "6384": ["", "", "", ""], "6385": ["", "", "", ""], "6386": ["", "", "", ""], "6387": ["", "", "", ""], "6388": ["", "", "", ""], "6389": ["", "", "", ""], "6390": ["", "", "", ""], "6391": ["", "", "", ""], "6392": ["", "", "", ""], "6393": ["", "", "", ""], "6394": ["", "", "", ""], "6395": ["", "", "", ""], "6396": ["", "", "", ""], "6397": ["", "", "", ""], "6398": ["", "", "", ""], "6399": ["", "", "", ""], "6400": ["", "", "", ""], "6401": ["", "", "", ""], "6402": ["", "", "", ""], "6403": ["", "", "", ""], "6404": ["", "", "", ""], "6405": ["", "", "", ""], "6406": ["", "", "", ""], "6407": ["", "", "", ""], "6408": ["", "", "", ""], "6409": ["", "", "", ""], "6410": ["", "", "", ""], "6411": ["", "", "", ""], "6412": ["", "", "", ""], "6413": ["", "", "", ""], "6414": ["", "", "", ""], "6415": ["", "", "", ""], "6416": ["", "", "", ""], "6417": ["", "", "", ""], "6418": ["", "", "", ""], "6419": ["", "", "", ""], "6420": ["", "", "", ""], "6421": ["", "", "", ""], "6422": ["", "", "", ""], "6423": ["", "", "", ""], "6424": ["", "", "", ""], "6425": ["", "", "", ""], "6426": ["", "", "", ""], "6427": ["", "", "", ""], "6428": ["", "", "", ""], "6429": ["", "", "", ""], "6430": ["", "", "", ""], "6431": ["", "", "", ""], "6432": ["", "", "", ""], "6433": ["", "", "", ""], "6434": ["", "", "", ""], "6435": ["", "", "", ""], "6436": ["", "", "", ""], "6437": ["", "", "", ""], "6438": ["", "", "", ""], "6439": ["", "", "", ""], "6440": ["", "", "", ""], "6441": ["", "", "", ""], "6442": ["", "", "", ""], "6443": ["", "", "", ""], "6444": ["", "", "", ""], "6445": ["", "", "", ""], "6446": ["", "", "", ""], "6447": ["", "", "", ""], "6448": ["", "", "", ""], "6449": ["", "", "", ""], "6450": ["", "", "", ""], "6451": ["", "", "", ""], "6452": ["", "", "", ""], "6453": ["", "", "", ""], "6454": ["", "", "", ""], "6455": ["", "", "", ""], "6456": ["", "", "", ""], "6457": ["", "", "", ""], "6458": ["", "", "", ""], "6459": ["", "", "", ""], "6460": ["", "", "", ""], "6461": ["", "", "", ""], "6462": ["", "", "", ""], "6463": ["", "", "", ""], "6464": ["", "", "", ""], "6465": ["", "", "", ""], "6466": ["", "", "", ""], "6467": ["", "", "", ""], "6468": ["", "", "", ""], "6469": ["", "", "", ""], "6470": ["", "", "", ""], "6471": ["", "", "", ""], "6472": ["", "", "", ""], "6473": ["", "", "", ""], "6474": ["", "", "", ""], "6475": ["", "", "", ""], "6476": ["", "", "", ""], "6477": ["", "", "", ""], "6478": ["", "", "", ""], "6479": ["", "", "", ""], "6480": ["", "", "", ""], "6481": ["", "", "", ""], "6482": ["", "", "", ""], "6483": ["", "", "", ""], "6484": ["", "", "", ""], "6485": ["", "", "", ""], "6486": ["", "", "", ""], "6487": ["", "", "", ""], "6488": ["", "", "", ""], "6489": ["", "", "", ""], "6490": ["", "", "", ""], "6491": ["", "", "", ""], "6492": ["", "", "", ""], "6493": ["", "", "", ""], "6494": ["", "", "", ""], "6495": ["", "", "", ""], "6496": ["", "", "", ""], "6497": ["", "", "", ""], "6498": ["", "", "", ""], "6499": ["", "", "", ""], "6500": ["", "", "", ""], "6501": ["", "", "", ""], "6502": ["", "", "", ""], "6503": ["", "", "", ""], "6504": ["", "", "", ""], "6505": ["", "", "", ""], "6506": ["", "", "", ""], "6507": ["", "", "", ""], "6508": ["", "", "", ""], "6509": ["", "", "", ""], "6510": ["", "", "", ""], "6511": ["", "", "", ""], "6512": ["", "", "", ""], "6513": ["", "", "", ""], "6514": ["", "", "", ""], "6515": ["", "", "", ""], "6516": ["", "", "", ""], "6517": ["", "", "", ""], "6518": ["", "", "", ""], "6519": ["", "", "", ""], "6520": ["", "", "", ""], "6521": ["", "", "", ""], "6522": ["", "", "", ""], "6523": ["", "", "", ""], "6524": ["", "", "", ""], "6525": ["", "", "", ""], "6526": ["", "", "", ""], "6527": ["", "", "", ""], "6528": ["", "", "", ""], "6529": ["", "", "", ""], "6530": ["", "", "", ""], "6531": ["", "", "", ""], "6532": ["", "", "", ""], "6533": ["", "", "", ""], "6534": ["", "", "", ""], "6535": ["", "", "", ""], "6536": ["", "", "", ""], "6537": ["", "", "", ""], "6538": ["", "", "", ""], "6539": ["", "", "", ""], "6540": ["", "", "", ""], "6541": ["", "", "", ""], "6542": ["", "", "", ""], "6543": ["", "", "", ""], "6544": ["", "", "", ""], "6545": ["", "", "", ""], "6546": ["", "", "", ""], "6547": ["", "", "", ""], "6548": ["", "", "", ""], "6549": ["", "", "", ""], "6550": ["", "", "", ""], "6551": ["", "", "", ""], "6552": ["", "", "", ""], "6553": ["", "", "", ""], "6554": ["", "", "", ""], "6555": ["", "", "", ""], "6556": ["", "", "", ""], "6557": ["", "", "", ""], "6558": ["", "", "", ""], "6559": ["", "", "", ""], "6560": ["", "", "", ""], "6561": ["", "", "", ""], "6562": ["", "", "", ""], "6563": ["", "", "", ""], "6564": ["", "", "", ""], "6565": ["", "", "", ""], "6566": ["", "", "", ""], "6567": ["", "", "", ""], "6568": ["", "", "", ""], "6569": ["", "", "", ""], "6570": ["", "", "", ""], "6571": ["", "", "", ""], "6572": ["", "", "", ""], "6573": ["", "", "", ""], "6574": ["", "", "", ""], "6575": ["", "", "", ""], "6576": ["", "", "", ""], "6577": ["", "", "", ""], "6578": ["", "", "", ""], "6579": ["", "", "", ""], "6580": ["", "", "", ""], "6581": ["", "", "", ""], "6582": ["", "", "", ""], "6583": ["", "", "", ""], "6584": ["", "", "", ""], "6585": ["", "", "", ""], "6586": ["", "", "", ""], "6587": ["", "", "", ""], "6588": ["", "", "", ""], "6589": ["", "", "", ""], "6590": ["", "", "", ""], "6591": ["", "", "", ""], "6592": ["", "", "", ""], "6593": ["", "", "", ""], "6594": ["", "", "", ""], "6595": ["", "", "", ""], "6596": ["", "", "", ""], "6597": ["", "", "", ""], "6598": ["", "", "", ""], "6599": ["", "", "", ""], "6600": ["", "", "", ""], "6601": ["", "", "", ""], "6602": ["", "", "", ""], "6603": ["", "", "", ""], "6604": ["", "", "", ""], "6605": ["", "", "", ""], "6606": ["", "", "", ""], "6607": ["", "", "", ""], "6608": ["", "", "", ""], "6609": ["", "", "", ""], "6610": ["", "", "", ""], "6611": ["", "", "", ""], "6612": ["", "", "", ""], "6613": ["", "", "", ""], "6614": ["", "", "", ""], "6615": ["", "", "", ""], "6616": ["", "", "", ""], "6617": ["", "", "", ""], "6618": ["", "", "", ""], "6619": ["", "", "", ""], "6620": ["", "", "", ""], "6621": ["", "", "", ""], "6622": ["", "", "", ""], "6623": ["", "", "", ""], "6624": ["", "", "", ""], "6625": ["", "", "", ""], "6626": ["", "", "", ""], "6627": ["", "", "", ""], "6628": ["", "", "", ""], "6629": ["", "", "", ""], "6630": ["", "", "", ""], "6631": ["", "", "", ""], "6632": ["", "", "", ""], "6633": ["", "", "", ""], "6634": ["", "", "", ""], "6635": ["", "", "", ""], "6636": ["", "", "", ""], "6637": ["", "", "", ""], "6638": ["", "", "", ""], "6639": ["", "", "", ""], "6640": ["", "", "", ""], "6641": ["", "", "", ""], "6642": ["", "", "", ""], "6643": ["", "", "", ""], "6644": ["", "", "", ""], "6645": ["", "", "", ""], "6646": ["", "", "", ""], "6647": ["", "", "", ""], "6648": ["", "", "", ""], "6649": ["", "", "", ""], "6650": ["", "", "", ""], "6651": ["", "", "", ""], "6652": ["", "", "", ""], "6653": ["", "", "", ""], "6654": ["", "", "", ""], "6655": ["", "", "", ""], "6656": ["", "", "", ""], "6657": ["", "", "", ""], "6658": ["", "", "", ""], "6659": ["", "", "", ""], "6660": ["", "", "", ""], "6661": ["", "", "", ""], "6662": ["", "", "", ""], "6663": ["", "", "", ""], "6664": ["", "", "", ""], "6665": ["", "", "", ""], "6666": ["", "", "", ""], "6667": ["", "", "", ""], "6668": ["", "", "", ""], "6669": ["", "", "", ""], "6670": ["", "", "", ""], "6671": ["", "", "", ""], "6672": ["", "", "", ""], "6673": ["", "", "", ""], "6674": ["", "", "", ""], "6675": ["", "", "", ""], "6676": ["", "", "", ""], "6677": ["", "", "", ""], "6678": ["", "", "", ""], "6679": ["", "", "", ""], "6680": ["", "", "", ""], "6681": ["", "", "", ""], "6682": ["", "", "", ""], "6683": ["", "", "", ""], "6684": ["", "", "", ""], "6685": ["", "", "", ""], "6686": ["", "", "", ""], "6687": ["", "", "", ""], "6688": ["", "", "", ""], "6689": ["", "", "", ""], "6690": ["", "", "", ""], "6691": ["", "", "", ""], "6692": ["", "", "", ""], "6693": ["", "", "", ""], "6694": ["", "", "", ""], "6695": ["", "", "", ""], "6696": ["", "", "", ""], "6697": ["", "", "", ""], "6698": ["", "", "", ""], "6699": ["", "", "", ""], "6700": ["", "", "", ""], "6701": ["", "", "", ""], "6702": ["", "", "", ""], "6703": ["", "", "", ""], "6704": ["", "", "", ""], "6705": ["", "", "", ""], "6706": ["", "", "", ""], "6707": ["", "", "", ""], "6708": ["", "", "", ""], "6709": ["", "", "", ""], "6710": ["", "", "", ""], "6711": ["", "", "", ""], "6712": ["", "", "", ""], "6713": ["", "", "", ""], "6714": ["", "", "", ""], "6715": ["", "", "", ""], "6716": ["", "", "", ""], "6717": ["", "", "", ""], "6718": ["", "", "", ""], "6719": ["", "", "", ""], "6720": ["", "", "", ""], "6721": ["", "", "", ""], "6722": ["", "", "", ""], "6723": ["", "", "", ""], "6724": ["", "", "", ""], "6725": ["", "", "", ""], "6726": ["", "", "", ""], "6727": ["", "", "", ""], "6728": ["", "", "", ""], "6729": ["", "", "", ""], "6730": ["", "", "", ""], "6731": ["", "", "", ""], "6732": ["", "", "", ""], "6733": ["", "", "", ""], "6734": ["", "", "", ""], "6735": ["", "", "", ""], "6736": ["", "", "", ""], "6737": ["", "", "", ""], "6738": ["", "", "", ""], "6739": ["", "", "", ""], "6740": ["", "", "", ""], "6741": ["", "", "", ""], "6742": ["", "", "", ""], "6743": ["", "", "", ""], "6744": ["", "", "", ""], "6745": ["", "", "", ""], "6746": ["", "", "", ""], "6747": ["", "", "", ""], "6748": ["", "", "", ""], "6749": ["", "", "", ""], "6750": ["", "", "", ""], "6751": ["", "", "", ""], "6752": ["", "", "", ""], "6753": ["", "", "", ""], "6754": ["", "", "", ""], "6755": ["", "", "", ""], "6756": ["", "", "", ""], "6757": ["", "", "", ""], "6758": ["", "", "", ""], "6759": ["", "", "", ""], "6760": ["", "", "", ""], "6761": ["", "", "", ""], "6762": ["", "", "", ""], "6763": ["", "", "", ""], "6764": ["", "", "", ""], "6765": ["", "", "", ""], "6766": ["", "", "", ""], "6767": ["", "", "", ""], "6768": ["", "", "", ""], "6769": ["", "", "", ""], "6770": ["", "", "", ""], "6771": ["", "", "", ""], "6772": ["", "", "", ""], "6773": ["", "", "", ""], "6774": ["", "", "", ""], "6775": ["", "", "", ""], "6776": ["", "", "", ""], "6777": ["", "", "", ""], "6778": ["", "", "", ""], "6779": ["", "", "", ""], "6780": ["", "", "", ""], "6781": ["", "", "", ""], "6782": ["", "", "", ""], "6783": ["", "", "", ""], "6784": ["", "", "", ""], "6785": ["", "", "", ""], "6786": ["", "", "", ""], "6787": ["", "", "", ""], "6788": ["", "", "", ""], "6789": ["", "", "", ""], "6790": ["", "", "", ""], "6791": ["", "", "", ""], "6792": ["", "", "", ""], "6793": ["", "", "", ""], "6794": ["", "", "", ""], "6795": ["", "", "", ""], "6796": ["", "", "", ""], "6797": ["", "", "", ""], "6798": ["", "", "", ""], "6799": ["", "", "", ""], "6800": ["", "", "", ""], "6801": ["", "", "", ""], "6802": ["", "", "", ""], "6803": ["", "", "", ""], "6804": ["", "", "", ""], "6805": ["", "", "", ""], "6806": ["", "", "", ""], "6807": ["", "", "", ""], "6808": ["", "", "", ""], "6809": ["", "", "", ""], "6810": ["", "", "", ""], "6811": ["", "", "", ""], "6812": ["", "", "", ""], "6813": ["", "", "", ""], "6814": ["", "", "", ""], "6815": ["", "", "", ""], "6816": ["", "", "", ""], "6817": ["", "", "", ""], "6818": ["", "", "", ""], "6819": ["", "", "", ""], "6820": ["", "", "", ""], "6821": ["", "", "", ""], "6822": ["", "", "", ""], "6823": ["", "", "", ""], "6824": ["", "", "", ""], "6825": ["", "", "", ""], "6826": ["", "", "", ""], "6827": ["", "", "", ""], "6828": ["", "", "", ""], "6829": ["", "", "", ""], "6830": ["", "", "", ""], "6831": ["", "", "", ""], "6832": ["", "", "", ""], "6833": ["", "", "", ""], "6834": ["", "", "", ""], "6835": ["", "", "", ""], "6836": ["", "", "", ""], "6837": ["", "", "", ""], "6838": ["", "", "", ""], "6839": ["", "", "", ""], "6840": ["", "", "", ""], "6841": ["", "", "", ""], "6842": ["", "", "", ""], "6843": ["", "", "", ""], "6844": ["", "", "", ""], "6845": ["", "", "", ""], "6846": ["", "", "", ""], "6847": ["", "", "", ""], "6848": ["", "", "", ""], "6849": ["", "", "", ""], "6850": ["", "", "", ""], "6851": ["", "", "", ""], "6852": ["", "", "", ""], "6853": ["", "", "", ""], "6854": ["", "", "", ""], "6855": ["", "", "", ""], "6856": ["", "", "", ""], "6857": ["", "", "", ""], "6858": ["", "", "", ""], "6859": ["", "", "", ""], "6860": ["", "", "", ""], "6861": ["", "", "", ""], "6862": ["", "", "", ""], "6863": ["", "", "", ""], "6864": ["", "", "", ""], "6865": ["", "", "", ""], "6866": ["", "", "", ""], "6867": ["", "", "", ""], "6868": ["", "", "", ""], "6869": ["", "", "", ""], "6870": ["", "", "", ""], "6871": ["", "", "", ""], "6872": ["", "", "", ""], "6873": ["", "", "", ""], "6874": ["", "", "", ""], "6875": ["", "", "", ""], "6876": ["", "", "", ""], "6877": ["", "", "", ""], "6878": ["", "", "", ""], "6879": ["", "", "", ""], "6880": ["", "", "", ""], "6881": ["", "", "", ""], "6882": ["", "", "", ""], "6883": ["", "", "", ""], "6884": ["", "", "", ""], "6885": ["", "", "", ""], "6886": ["", "", "", ""], "6887": ["", "", "", ""], "6888": ["", "", "", ""], "6889": ["", "", "", ""], "6890": ["", "", "", ""], "6891": ["", "", "", ""], "6892": ["", "", "", ""], "6893": ["", "", "", ""], "6894": ["", "", "", ""], "6895": ["", "", "", ""], "6896": ["", "", "", ""], "6897": ["", "", "", ""], "6898": ["", "", "", ""], "6899": ["", "", "", ""], "6900": ["", "", "", ""], "6901": ["", "", "", ""], "6902": ["", "", "", ""], "6903": ["", "", "", ""], "6904": ["", "", "", ""], "6905": ["", "", "", ""], "6906": ["", "", "", ""], "6907": ["", "", "", ""], "6908": ["", "", "", ""], "6909": ["", "", "", ""], "6910": ["", "", "", ""], "6911": ["", "", "", ""], "6912": ["", "", "", ""], "6913": ["", "", "", ""], "6914": ["", "", "", ""], "6915": ["", "", "", ""], "6916": ["", "", "", ""], "6917": ["", "", "", ""], "6918": ["", "", "", ""], "6919": ["", "", "", ""], "6920": ["", "", "", ""], "6921": ["", "", "", ""], "6922": ["", "", "", ""], "6923": ["", "", "", ""], "6924": ["", "", "", ""], "6925": ["", "", "", ""], "6926": ["", "", "", ""], "6927": ["", "", "", ""], "6928": ["", "", "", ""], "6929": ["", "", "", ""], "6930": ["", "", "", ""], "6931": ["", "", "", ""], "6932": ["", "", "", ""], "6933": ["", "", "", ""], "6934": ["", "", "", ""], "6935": ["", "", "", ""], "6936": ["", "", "", ""], "6937": ["", "", "", ""], "6938": ["", "", "", ""], "6939": ["", "", "", ""], "6940": ["", "", "", ""], "6941": ["", "", "", ""], "6942": ["", "", "", ""], "6943": ["", "", "", ""], "6944": ["", "", "", ""], "6945": ["", "", "", ""], "6946": ["", "", "", ""], "6947": ["", "", "", ""], "6948": ["", "", "", ""], "6949": ["", "", "", ""], "6950": ["", "", "", ""], "6951": ["", "", "", ""], "6952": ["", "", "", ""], "6953": ["", "", "", ""], "6954": ["", "", "", ""], "6955": ["", "", "", ""], "6956": ["", "", "", ""], "6957": ["", "", "", ""], "6958": ["", "", "", ""], "6959": ["", "", "", ""], "6960": ["", "", "", ""], "6961": ["", "", "", ""], "6962": ["", "", "", ""], "6963": ["", "", "", ""], "6964": ["", "", "", ""], "6965": ["", "", "", ""], "6966": ["", "", "", ""], "6967": ["", "", "", ""], "6968": ["", "", "", ""], "6969": ["", "", "", ""], "6970": ["", "", "", ""], "6971": ["", "", "", ""], "6972": ["", "", "", ""], "6973": ["", "", "", ""], "6974": ["", "", "", ""], "6975": ["", "", "", ""], "6976": ["", "", "", ""], "6977": ["", "", "", ""], "6978": ["", "", "", ""], "6979": ["", "", "", ""], "6980": ["", "", "", ""], "6981": ["", "", "", ""], "6982": ["", "", "", ""], "6983": ["", "", "", ""], "6984": ["", "", "", ""], "6985": ["", "", "", ""], "6986": ["", "", "", ""], "6987": ["", "", "", ""], "6988": ["", "", "", ""], "6989": ["", "", "", ""], "6990": ["", "", "", ""], "6991": ["", "", "", ""], "6992": ["", "", "", ""], "6993": ["", "", "", ""], "6994": ["", "", "", ""], "6995": ["", "", "", ""], "6996": ["", "", "", ""], "6997": ["", "", "", ""], "6998": ["", "", "", ""], "6999": ["", "", "", ""], "7000": ["", "", "", ""], "7001": ["", "", "", ""], "7002": ["", "", "", ""], "7003": ["", "", "", ""], "7004": ["", "", "", ""], "7005": ["", "", "", ""], "7006": ["", "", "", ""], "7007": ["", "", "", ""], "7008": ["", "", "", ""], "7009": ["", "", "", ""], "7010": ["", "", "", ""], "7011": ["", "", "", ""], "7012": ["", "", "", ""], "7013": ["", "", "", ""], "7014": ["", "", "", ""], "7015": ["", "", "", ""], "7016": ["", "", "", ""], "7017": ["", "", "", ""], "7018": ["", "", "", ""], "7019": ["", "", "", ""], "7020": ["", "", "", ""], "7021": ["", "", "", ""], "7022": ["", "", "", ""], "7023": ["", "", "", ""], "7024": ["", "", "", ""], "7025": ["", "", "", ""], "7026": ["", "", "", ""], "7027": ["", "", "", ""], "7028": ["", "", "", ""], "7029": ["", "", "", ""], "7030": ["", "", "", ""], "7031": ["", "", "", ""], "7032": ["", "", "", ""], "7033": ["", "", "", ""], "7034": ["", "", "", ""], "7035": ["", "", "", ""], "7036": ["", "", "", ""], "7037": ["", "", "", ""], "7038": ["", "", "", ""], "7039": ["", "", "", ""], "7040": ["", "", "", ""], "7041": ["", "", "", ""], "7042": ["", "", "", ""], "7043": ["", "", "", ""], "7044": ["", "", "", ""], "7045": ["", "", "", ""], "7046": ["", "", "", ""], "7047": ["", "", "", ""], "7048": ["", "", "", ""], "7049": ["", "", "", ""], "7050": ["", "", "", ""], "7051": ["", "", "", ""], "7052": ["", "", "", ""], "7053": ["", "", "", ""], "7054": ["", "", "", ""], "7055": ["", "", "", ""], "7056": ["", "", "", ""], "7057": ["", "", "", ""], "7058": ["", "", "", ""], "7059": ["", "", "", ""], "7060": ["", "", "", ""], "7061": ["", "", "", ""], "7062": ["", "", "", ""], "7063": ["", "", "", ""], "7064": ["", "", "", ""], "7065": ["", "", "", ""], "7066": ["", "", "", ""], "7067": ["", "", "", ""], "7068": ["", "", "", ""], "7069": ["", "", "", ""], "7070": ["", "", "", ""], "7071": ["", "", "", ""], "7072": ["", "", "", ""], "7073": ["", "", "", ""], "7074": ["", "", "", ""], "7075": ["", "", "", ""], "7076": ["", "", "", ""], "7077": ["", "", "", ""], "7078": ["", "", "", ""], "7079": ["", "", "", ""], "7080": ["", "", "", ""], "7081": ["", "", "", ""], "7082": ["", "", "", ""], "7083": ["", "", "", ""], "7084": ["", "", "", ""], "7085": ["", "", "", ""], "7086": ["", "", "", ""], "7087": ["", "", "", ""], "7088": ["", "", "", ""], "7089": ["", "", "", ""], "7090": ["", "", "", ""], "7091": ["", "", "", ""], "7092": ["", "", "", ""], "7093": ["", "", "", ""], "7094": ["", "", "", ""], "7095": ["", "", "", ""], "7096": ["", "", "", ""], "7097": ["", "", "", ""], "7098": ["", "", "", ""], "7099": ["", "", "", ""], "7100": ["", "", "", ""], "7101": ["", "", "", ""], "7102": ["", "", "", ""], "7103": ["", "", "", ""], "7104": ["", "", "", ""], "7105": ["", "", "", ""], "7106": ["", "", "", ""], "7107": ["", "", "", ""], "7108": ["", "", "", ""], "7109": ["", "", "", ""], "7110": ["", "", "", ""], "7111": ["", "", "", ""], "7112": ["", "", "", ""], "7113": ["", "", "", ""], "7114": ["", "", "", ""], "7115": ["", "", "", ""], "7116": ["", "", "", ""], "7117": ["", "", "", ""], "7118": ["", "", "", ""], "7119": ["", "", "", ""], "7120": ["", "", "", ""], "7121": ["", "", "", ""], "7122": ["", "", "", ""], "7123": ["", "", "", ""], "7124": ["", "", "", ""], "7125": ["", "", "", ""], "7126": ["", "", "", ""], "7127": ["", "", "", ""], "7128": ["", "", "", ""], "7129": ["", "", "", ""], "7130": ["", "", "", ""], "7131": ["", "", "", ""], "7132": ["", "", "", ""], "7133": ["", "", "", ""], "7134": ["", "", "", ""], "7135": ["", "", "", ""], "7136": ["", "", "", ""], "7137": ["", "", "", ""], "7138": ["", "", "", ""], "7139": ["", "", "", ""], "7140": ["", "", "", ""], "7141": ["", "", "", ""], "7142": ["", "", "", ""], "7143": ["", "", "", ""], "7144": ["", "", "", ""], "7145": ["", "", "", ""], "7146": ["", "", "", ""], "7147": ["", "", "", ""], "7148": ["", "", "", ""], "7149": ["", "", "", ""], "7150": ["", "", "", ""], "7151": ["", "", "", ""], "7152": ["", "", "", ""], "7153": ["", "", "", ""], "7154": ["", "", "", ""], "7155": ["", "", "", ""], "7156": ["", "", "", ""], "7157": ["", "", "", ""], "7158": ["", "", "", ""], "7159": ["", "", "", ""], "7160": ["", "", "", ""], "7161": ["", "", "", ""], "7162": ["", "", "", ""], "7163": ["", "", "", ""], "7164": ["", "", "", ""], "7165": ["", "", "", ""], "7166": ["", "", "", ""], "7167": ["", "", "", ""], "7168": ["", "", "", ""], "7169": ["", "", "", ""], "7170": ["", "", "", ""], "7171": ["", "", "", ""], "7172": ["", "", "", ""], "7173": ["", "", "", ""], "7174": ["", "", "", ""], "7175": ["", "", "", ""], "7176": ["", "", "", ""], "7177": ["", "", "", ""], "7178": ["", "", "", ""], "7179": ["", "", "", ""], "7180": ["", "", "", ""], "7181": ["", "", "", ""], "7182": ["", "", "", ""], "7183": ["", "", "", ""], "7184": ["", "", "", ""], "7185": ["", "", "", ""], "7186": ["", "", "", ""], "7187": ["", "", "", ""], "7188": ["", "", "", ""], "7189": ["", "", "", ""], "7190": ["", "", "", ""], "7191": ["", "", "", ""], "7192": ["", "", "", ""], "7193": ["", "", "", ""], "7194": ["", "", "", ""], "7195": ["", "", "", ""], "7196": ["", "", "", ""], "7197": ["", "", "", ""], "7198": ["", "", "", ""], "7199": ["", "", "", ""], "7200": ["", "", "", ""], "7201": ["", "", "", ""], "7202": ["", "", "", ""], "7203": ["", "", "", ""], "7204": ["", "", "", ""], "7205": ["", "", "", ""], "7206": ["", "", "", ""], "7207": ["", "", "", ""], "7208": ["", "", "", ""], "7209": ["", "", "", ""], "7210": ["", "", "", ""], "7211": ["", "", "", ""], "7212": ["", "", "", ""], "7213": ["", "", "", ""], "7214": ["", "", "", ""], "7215": ["", "", "", ""], "7216": ["", "", "", ""], "7217": ["", "", "", ""], "7218": ["", "", "", ""], "7219": ["", "", "", ""], "7220": ["", "", "", ""], "7221": ["", "", "", ""], "7222": ["", "", "", ""], "7223": ["", "", "", ""], "7224": ["", "", "", ""], "7225": ["", "", "", ""], "7226": ["", "", "", ""], "7227": ["", "", "", ""], "7228": ["", "", "", ""], "7229": ["", "", "", ""], "7230": ["", "", "", ""], "7231": ["", "", "", ""], "7232": ["", "", "", ""], "7233": ["", "", "", ""], "7234": ["", "", "", ""], "7235": ["", "", "", ""], "7236": ["", "", "", ""], "7237": ["", "", "", ""], "7238": ["", "", "", ""], "7239": ["", "", "", ""], "7240": ["", "", "", ""], "7241": ["", "", "", ""], "7242": ["", "", "", ""], "7243": ["", "", "", ""], "7244": ["", "", "", ""], "7245": ["", "", "", ""], "7246": ["", "", "", ""], "7247": ["", "", "", ""], "7248": ["", "", "", ""], "7249": ["", "", "", ""], "7250": ["", "", "", ""], "7251": ["", "", "", ""], "7252": ["", "", "", ""], "7253": ["", "", "", ""], "7254": ["", "", "", ""], "7255": ["", "", "", ""], "7256": ["", "", "", ""], "7257": ["", "", "", ""], "7258": ["", "", "", ""], "7259": ["", "", "", ""], "7260": ["", "", "", ""], "7261": ["", "", "", ""], "7262": ["", "", "", ""], "7263": ["", "", "", ""], "7264": ["", "", "", ""], "7265": ["", "", "", ""], "7266": ["", "", "", ""], "7267": ["", "", "", ""], "7268": ["", "", "", ""], "7269": ["", "", "", ""], "7270": ["", "", "", ""], "7271": ["", "", "", ""], "7272": ["", "", "", ""], "7273": ["", "", "", ""], "7274": ["", "", "", ""], "7275": ["", "", "", ""], "7276": ["", "", "", ""], "7277": ["", "", "", ""], "7278": ["", "", "", ""], "7279": ["", "", "", ""], "7280": ["", "", "", ""], "7281": ["", "", "", ""], "7282": ["", "", "", ""], "7283": ["", "", "", ""], "7284": ["", "", "", ""], "7285": ["", "", "", ""], "7286": ["", "", "", ""], "7287": ["", "", "", ""], "7288": ["", "", "", ""], "7289": ["", "", "", ""], "7290": ["", "", "", ""], "7291": ["", "", "", ""], "7292": ["", "", "", ""], "7293": ["", "", "", ""], "7294": ["", "", "", ""], "7295": ["", "", "", ""], "7296": ["", "", "", ""], "7297": ["", "", "", ""], "7298": ["", "", "", ""], "7299": ["", "", "", ""], "7300": ["", "", "", ""], "7301": ["", "", "", ""], "7302": ["", "", "", ""], "7303": ["", "", "", ""], "7304": ["", "", "", ""], "7305": ["", "", "", ""], "7306": ["", "", "", ""], "7307": ["", "", "", ""], "7308": ["", "", "", ""], "7309": ["", "", "", ""], "7310": ["", "", "", ""], "7311": ["", "", "", ""], "7312": ["", "", "", ""], "7313": ["", "", "", ""], "7314": ["", "", "", ""], "7315": ["", "", "", ""], "7316": ["", "", "", ""], "7317": ["", "", "", ""], "7318": ["", "", "", ""], "7319": ["", "", "", ""], "7320": ["", "", "", ""], "7321": ["", "", "", ""], "7322": ["", "", "", ""], "7323": ["", "", "", ""], "7324": ["", "", "", ""], "7325": ["", "", "", ""], "7326": ["", "", "", ""], "7327": ["", "", "", ""], "7328": ["", "", "", ""], "7329": ["", "", "", ""], "7330": ["", "", "", ""], "7331": ["", "", "", ""], "7332": ["", "", "", ""], "7333": ["", "", "", ""], "7334": ["", "", "", ""], "7335": ["", "", "", ""], "7336": ["", "", "", ""], "7337": ["", "", "", ""], "7338": ["", "", "", ""], "7339": ["", "", "", ""], "7340": ["", "", "", ""], "7341": ["", "", "", ""], "7342": ["", "", "", ""], "7343": ["", "", "", ""], "7344": ["", "", "", ""], "7345": ["", "", "", ""], "7346": ["", "", "", ""], "7347": ["", "", "", ""], "7348": ["", "", "", ""], "7349": ["", "", "", ""], "7350": ["", "", "", ""], "7351": ["", "", "", ""], "7352": ["", "", "", ""], "7353": ["", "", "", ""], "7354": ["", "", "", ""], "7355": ["", "", "", ""], "7356": ["", "", "", ""], "7357": ["", "", "", ""], "7358": ["", "", "", ""], "7359": ["", "", "", ""], "7360": ["", "", "", ""], "7361": ["", "", "", ""], "7362": ["", "", "", ""], "7363": ["", "", "", ""], "7364": ["", "", "", ""], "7365": ["", "", "", ""], "7366": ["", "", "", ""], "7367": ["", "", "", ""], "7368": ["", "", "", ""], "7369": ["", "", "", ""], "7370": ["", "", "", ""], "7371": ["", "", "", ""], "7372": ["", "", "", ""], "7373": ["", "", "", ""], "7374": ["", "", "", ""], "7375": ["", "", "", ""], "7376": ["", "", "", ""], "7377": ["", "", "", ""], "7378": ["", "", "", ""], "7379": ["", "", "", ""], "7380": ["", "", "", ""], "7381": ["", "", "", ""], "7382": ["", "", "", ""], "7383": ["", "", "", ""], "7384": ["", "", "", ""], "7385": ["", "", "", ""], "7386": ["", "", "", ""], "7387": ["", "", "", ""], "7388": ["", "", "", ""], "7389": ["", "", "", ""], "7390": ["", "", "", ""], "7391": ["", "", "", ""], "7392": ["", "", "", ""], "7393": ["", "", "", ""], "7394": ["", "", "", ""], "7395": ["", "", "", ""], "7396": ["", "", "", ""], "7397": ["", "", "", ""], "7398": ["", "", "", ""], "7399": ["", "", "", ""], "7400": ["", "", "", ""], "7401": ["", "", "", ""], "7402": ["", "", "", ""], "7403": ["", "", "", ""], "7404": ["", "", "", ""], "7405": ["", "", "", ""], "7406": ["", "", "", ""], "7407": ["", "", "", ""], "7408": ["", "", "", ""], "7409": ["", "", "", ""], "7410": ["", "", "", ""], "7411": ["", "", "", ""], "7412": ["", "", "", ""], "7413": ["", "", "", ""], "7414": ["", "", "", ""], "7415": ["", "", "", ""], "7416": ["", "", "", ""], "7417": ["", "", "", ""], "7418": ["", "", "", ""], "7419": ["", "", "", ""], "7420": ["", "", "", ""], "7421": ["", "", "", ""], "7422": ["", "", "", ""], "7423": ["", "", "", ""], "7424": ["", "", "", ""], "7425": ["", "", "", ""], "7426": ["", "", "", ""], "7427": ["", "", "", ""], "7428": ["", "", "", ""], "7429": ["", "", "", ""], "7430": ["", "", "", ""], "7431": ["", "", "", ""], "7432": ["", "", "", ""], "7433": ["", "", "", ""], "7434": ["", "", "", ""], "7435": ["", "", "", ""], "7436": ["", "", "", ""], "7437": ["", "", "", ""], "7438": ["", "", "", ""], "7439": ["", "", "", ""], "7440": ["", "", "", ""], "7441": ["", "", "", ""], "7442": ["", "", "", ""], "7443": ["", "", "", ""], "7444": ["", "", "", ""], "7445": ["", "", "", ""], "7446": ["", "", "", ""], "7447": ["", "", "", ""], "7448": ["", "", "", ""], "7449": ["", "", "", ""], "7450": ["", "", "", ""], "7451": ["", "", "", ""], "7452": ["", "", "", ""], "7453": ["", "", "", ""], "7454": ["", "", "", ""], "7455": ["", "", "", ""], "7456": ["", "", "", ""], "7457": ["", "", "", ""], "7458": ["", "", "", ""], "7459": ["", "", "", ""], "7460": ["", "", "", ""], "7461": ["", "", "", ""], "7462": ["", "", "", ""], "7463": ["", "", "", ""], "7464": ["", "", "", ""], "7465": ["", "", "", ""], "7466": ["", "", "", ""], "7467": ["", "", "", ""], "7468": ["", "", "", ""], "7469": ["", "", "", ""], "7470": ["", "", "", ""], "7471": ["", "", "", ""], "7472": ["", "", "", ""], "7473": ["", "", "", ""], "7474": ["", "", "", ""], "7475": ["", "", "", ""], "7476": ["", "", "", ""], "7477": ["", "", "", ""], "7478": ["", "", "", ""], "7479": ["", "", "", ""], "7480": ["", "", "", ""], "7481": ["", "", "", ""], "7482": ["", "", "", ""], "7483": ["", "", "", ""], "7484": ["", "", "", ""], "7485": ["", "", "", ""], "7486": ["", "", "", ""], "7487": ["", "", "", ""], "7488": ["", "", "", ""], "7489": ["", "", "", ""], "7490": ["", "", "", ""], "7491": ["", "", "", ""], "7492": ["", "", "", ""], "7493": ["", "", "", ""], "7494": ["", "", "", ""], "7495": ["", "", "", ""], "7496": ["", "", "", ""], "7497": ["", "", "", ""], "7498": ["", "", "", ""], "7499": ["", "", "", ""], "7500": ["", "", "", ""], "7501": ["", "", "", ""], "7502": ["", "", "", ""], "7503": ["", "", "", ""], "7504": ["", "", "", ""], "7505": ["", "", "", ""], "7506": ["", "", "", ""], "7507": ["", "", "", ""], "7508": ["", "", "", ""], "7509": ["", "", "", ""], "7510": ["", "", "", ""], "7511": ["", "", "", ""], "7512": ["", "", "", ""], "7513": ["", "", "", ""], "7514": ["", "", "", ""], "7515": ["", "", "", ""], "7516": ["", "", "", ""], "7517": ["", "", "", ""], "7518": ["", "", "", ""], "7519": ["", "", "", ""], "7520": ["", "", "", ""], "7521": ["", "", "", ""], "7522": ["", "", "", ""], "7523": ["", "", "", ""], "7524": ["", "", "", ""], "7525": ["", "", "", ""], "7526": ["", "", "", ""], "7527": ["", "", "", ""], "7528": ["", "", "", ""], "7529": ["", "", "", ""], "7530": ["", "", "", ""], "7531": ["", "", "", ""], "7532": ["", "", "", ""], "7533": ["", "", "", ""], "7534": ["", "", "", ""], "7535": ["", "", "", ""], "7536": ["", "", "", ""], "7537": ["", "", "", ""], "7538": ["", "", "", ""], "7539": ["", "", "", ""], "7540": ["", "", "", ""], "7541": ["", "", "", ""], "7542": ["", "", "", ""], "7543": ["", "", "", ""], "7544": ["", "", "", ""], "7545": ["", "", "", ""], "7546": ["", "", "", ""], "7547": ["", "", "", ""], "7548": ["", "", "", ""], "7549": ["", "", "", ""], "7550": ["", "", "", ""], "7551": ["", "", "", ""], "7552": ["", "", "", ""], "7553": ["", "", "", ""], "7554": ["", "", "", ""], "7555": ["", "", "", ""], "7556": ["", "", "", ""], "7557": ["", "", "", ""], "7558": ["", "", "", ""], "7559": ["", "", "", ""], "7560": ["", "", "", ""], "7561": ["", "", "", ""], "7562": ["", "", "", ""], "7563": ["", "", "", ""], "7564": ["", "", "", ""], "7565": ["", "", "", ""], "7566": ["", "", "", ""], "7567": ["", "", "", ""], "7568": ["", "", "", ""], "7569": ["", "", "", ""], "7570": ["", "", "", ""], "7571": ["", "", "", ""], "7572": ["", "", "", ""], "7573": ["", "", "", ""], "7574": ["", "", "", ""], "7575": ["", "", "", ""], "7576": ["", "", "", ""], "7577": ["", "", "", ""], "7578": ["", "", "", ""], "7579": ["", "", "", ""], "7580": ["", "", "", ""], "7581": ["", "", "", ""], "7582": ["", "", "", ""], "7583": ["", "", "", ""], "7584": ["", "", "", ""], "7585": ["", "", "", ""], "7586": ["", "", "", ""], "7587": ["", "", "", ""], "7588": ["", "", "", ""], "7589": ["", "", "", ""], "7590": ["", "", "", ""], "7591": ["", "", "", ""], "7592": ["", "", "", ""], "7593": ["", "", "", ""], "7594": ["", "", "", ""], "7595": ["", "", "", ""], "7596": ["", "", "", ""], "7597": ["", "", "", ""], "7598": ["", "", "", ""], "7599": ["", "", "", ""], "7600": ["", "", "", ""], "7601": ["", "", "", ""], "7602": ["", "", "", ""], "7603": ["", "", "", ""], "7604": ["", "", "", ""], "7605": ["", "", "", ""], "7606": ["", "", "", ""], "7607": ["", "", "", ""], "7608": ["", "", "", ""], "7609": ["", "", "", ""], "7610": ["", "", "", ""], "7611": ["", "", "", ""], "7612": ["", "", "", ""], "7613": ["", "", "", ""], "7614": ["", "", "", ""], "7615": ["", "", "", ""], "7616": ["", "", "", ""], "7617": ["", "", "", ""], "7618": ["", "", "", ""], "7619": ["", "", "", ""], "7620": ["", "", "", ""], "7621": ["", "", "", ""], "7622": ["", "", "", ""], "7623": ["", "", "", ""], "7624": ["", "", "", ""], "7625": ["", "", "", ""], "7626": ["", "", "", ""], "7627": ["", "", "", ""], "7628": ["", "", "", ""], "7629": ["", "", "", ""], "7630": ["", "", "", ""], "7631": ["", "", "", ""], "7632": ["", "", "", ""], "7633": ["", "", "", ""], "7634": ["", "", "", ""], "7635": ["", "", "", ""], "7636": ["", "", "", ""], "7637": ["", "", "", ""], "7638": ["", "", "", ""], "7639": ["", "", "", ""], "7640": ["", "", "", ""], "7641": ["", "", "", ""], "7642": ["", "", "", ""], "7643": ["", "", "", ""], "7644": ["", "", "", ""], "7645": ["", "", "", ""], "7646": ["", "", "", ""], "7647": ["", "", "", ""], "7648": ["", "", "", ""], "7649": ["", "", "", ""], "7650": ["", "", "", ""], "7651": ["", "", "", ""], "7652": ["", "", "", ""], "7653": ["", "", "", ""], "7654": ["", "", "", ""], "7655": ["", "", "", ""], "7656": ["", "", "", ""], "7657": ["", "", "", ""], "7658": ["", "", "", ""], "7659": ["", "", "", ""], "7660": ["", "", "", ""], "7661": ["", "", "", ""], "7662": ["", "", "", ""], "7663": ["", "", "", ""], "7664": ["", "", "", ""], "7665": ["", "", "", ""], "7666": ["", "", "", ""], "7667": ["", "", "", ""], "7668": ["", "", "", ""], "7669": ["", "", "", ""], "7670": ["", "", "", ""], "7671": ["", "", "", ""], "7672": ["", "", "", ""], "7673": ["", "", "", ""], "7674": ["", "", "", ""], "7675": ["", "", "", ""], "7676": ["", "", "", ""], "7677": ["", "", "", ""], "7678": ["", "", "", ""], "7679": ["", "", "", ""], "7680": ["", "", "", ""], "7681": ["", "", "", ""], "7682": ["", "", "", ""], "7683": ["", "", "", ""], "7684": ["", "", "", ""], "7685": ["", "", "", ""], "7686": ["", "", "", ""], "7687": ["", "", "", ""], "7688": ["", "", "", ""], "7689": ["", "", "", ""], "7690": ["", "", "", ""], "7691": ["", "", "", ""], "7692": ["", "", "", ""], "7693": ["", "", "", ""], "7694": ["", "", "", ""], "7695": ["", "", "", ""], "7696": ["", "", "", ""], "7697": ["", "", "", ""], "7698": ["", "", "", ""], "7699": ["", "", "", ""], "7700": ["", "", "", ""], "7701": ["", "", "", ""], "7702": ["", "", "", ""], "7703": ["", "", "", ""], "7704": ["", "", "", ""], "7705": ["", "", "", ""], "7706": ["", "", "", ""], "7707": ["", "", "", ""], "7708": ["", "", "", ""], "7709": ["", "", "", ""], "7710": ["", "", "", ""], "7711": ["", "", "", ""], "7712": ["", "", "", ""], "7713": ["", "", "", ""], "7714": ["", "", "", ""], "7715": ["", "", "", ""], "7716": ["", "", "", ""], "7717": ["", "", "", ""], "7718": ["", "", "", ""], "7719": ["", "", "", ""], "7720": ["", "", "", ""], "7721": ["", "", "", ""], "7722": ["", "", "", ""], "7723": ["", "", "", ""], "7724": ["", "", "", ""], "7725": ["", "", "", ""], "7726": ["", "", "", ""], "7727": ["", "", "", ""], "7728": ["", "", "", ""], "7729": ["", "", "", ""], "7730": ["", "", "", ""], "7731": ["", "", "", ""], "7732": ["", "", "", ""], "7733": ["", "", "", ""], "7734": ["", "", "", ""], "7735": ["", "", "", ""], "7736": ["", "", "", ""], "7737": ["", "", "", ""], "7738": ["", "", "", ""], "7739": ["", "", "", ""], "7740": ["", "", "", ""], "7741": ["", "", "", ""], "7742": ["", "", "", ""], "7743": ["", "", "", ""], "7744": ["", "", "", ""], "7745": ["", "", "", ""], "7746": ["", "", "", ""], "7747": ["", "", "", ""], "7748": ["", "", "", ""], "7749": ["", "", "", ""], "7750": ["", "", "", ""], "7751": ["", "", "", ""], "7752": ["", "", "", ""], "7753": ["", "", "", ""], "7754": ["", "", "", ""], "7755": ["", "", "", ""], "7756": ["", "", "", ""], "7757": ["", "", "", ""], "7758": ["", "", "", ""], "7759": ["", "", "", ""], "7760": ["", "", "", ""], "7761": ["", "", "", ""], "7762": ["", "", "", ""], "7763": ["", "", "", ""], "7764": ["", "", "", ""], "7765": ["", "", "", ""], "7766": ["", "", "", ""], "7767": ["", "", "", ""], "7768": ["", "", "", ""], "7769": ["", "", "", ""], "7770": ["", "", "", ""], "7771": ["", "", "", ""], "7772": ["", "", "", ""], "7773": ["", "", "", ""], "7774": ["", "", "", ""], "7775": ["", "", "", ""], "7776": ["", "", "", ""], "7777": ["", "", "", ""], "7778": ["", "", "", ""], "7779": ["", "", "", ""], "7780": ["", "", "", ""], "7781": ["", "", "", ""], "7782": ["", "", "", ""], "7783": ["", "", "", ""], "7784": ["", "", "", ""], "7785": ["", "", "", ""], "7786": ["", "", "", ""], "7787": ["", "", "", ""], "7788": ["", "", "", ""], "7789": ["", "", "", ""], "7790": ["", "", "", ""], "7791": ["", "", "", ""], "7792": ["", "", "", ""], "7793": ["", "", "", ""], "7794": ["", "", "", ""], "7795": ["", "", "", ""], "7796": ["", "", "", ""], "7797": ["", "", "", ""], "7798": ["", "", "", ""], "7799": ["", "", "", ""], "7800": ["", "", "", ""], "7801": ["", "", "", ""], "7802": ["", "", "", ""], "7803": ["", "", "", ""], "7804": ["", "", "", ""], "7805": ["", "", "", ""], "7806": ["", "", "", ""], "7807": ["", "", "", ""], "7808": ["", "", "", ""], "7809": ["", "", "", ""], "7810": ["", "", "", ""], "7811": ["", "", "", ""], "7812": ["", "", "", ""], "7813": ["", "", "", ""], "7814": ["", "", "", ""], "7815": ["", "", "", ""], "7816": ["", "", "", ""], "7817": ["", "", "", ""], "7818": ["", "", "", ""], "7819": ["", "", "", ""], "7820": ["", "", "", ""], "7821": ["", "", "", ""], "7822": ["", "", "", ""], "7823": ["", "", "", ""], "7824": ["", "", "", ""], "7825": ["", "", "", ""], "7826": ["", "", "", ""], "7827": ["", "", "", ""], "7828": ["", "", "", ""], "7829": ["", "", "", ""], "7830": ["", "", "", ""], "7831": ["", "", "", ""], "7832": ["", "", "", ""], "7833": ["", "", "", ""], "7834": ["", "", "", ""], "7835": ["", "", "", ""], "7836": ["", "", "", ""], "7837": ["", "", "", ""], "7838": ["", "", "", ""], "7839": ["", "", "", ""], "7840": ["", "", "", ""], "7841": ["", "", "", ""], "7842": ["", "", "", ""], "7843": ["", "", "", ""], "7844": ["", "", "", ""], "7845": ["", "", "", ""], "7846": ["", "", "", ""], "7847": ["", "", "", ""], "7848": ["", "", "", ""], "7849": ["", "", "", ""], "7850": ["", "", "", ""], "7851": ["", "", "", ""], "7852": ["", "", "", ""], "7853": ["", "", "", ""], "7854": ["", "", "", ""], "7855": ["", "", "", ""], "7856": ["", "", "", ""], "7857": ["", "", "", ""], "7858": ["", "", "", ""], "7859": ["", "", "", ""], "7860": ["", "", "", ""], "7861": ["", "", "", ""], "7862": ["", "", "", ""], "7863": ["", "", "", ""], "7864": ["", "", "", ""], "7865": ["", "", "", ""], "7866": ["", "", "", ""], "7867": ["", "", "", ""], "7868": ["", "", "", ""], "7869": ["", "", "", ""], "7870": ["", "", "", ""], "7871": ["", "", "", ""], "7872": ["", "", "", ""], "7873": ["", "", "", ""], "7874": ["", "", "", ""], "7875": ["", "", "", ""], "7876": ["", "", "", ""], "7877": ["", "", "", ""], "7878": ["", "", "", ""], "7879": ["", "", "", ""], "7880": ["", "", "", ""], "7881": ["", "", "", ""], "7882": ["", "", "", ""], "7883": ["", "", "", ""], "7884": ["", "", "", ""], "7885": ["", "", "", ""], "7886": ["", "", "", ""], "7887": ["", "", "", ""], "7888": ["", "", "", ""], "7889": ["", "", "", ""], "7890": ["", "", "", ""], "7891": ["", "", "", ""], "7892": ["", "", "", ""], "7893": ["", "", "", ""], "7894": ["", "", "", ""], "7895": ["", "", "", ""], "7896": ["", "", "", ""], "7897": ["", "", "", ""], "7898": ["", "", "", ""], "7899": ["", "", "", ""], "7900": ["", "", "", ""], "7901": ["", "", "", ""], "7902": ["", "", "", ""], "7903": ["", "", "", ""], "7904": ["", "", "", ""], "7905": ["", "", "", ""], "7906": ["", "", "", ""], "7907": ["", "", "", ""], "7908": ["", "", "", ""], "7909": ["", "", "", ""], "7910": ["", "", "", ""], "7911": ["", "", "", ""], "7912": ["", "", "", ""], "7913": ["", "", "", ""], "7914": ["", "", "", ""], "7915": ["", "", "", ""], "7916": ["", "", "", ""], "7917": ["", "", "", ""], "7918": ["", "", "", ""], "7919": ["", "", "", ""], "7920": ["", "", "", ""], "7921": ["", "", "", ""], "7922": ["", "", "", ""], "7923": ["", "", "", ""], "7924": ["", "", "", ""], "7925": ["", "", "", ""], "7926": ["", "", "", ""], "7927": ["", "", "", ""], "7928": ["", "", "", ""], "7929": ["", "", "", ""], "7930": ["", "", "", ""], "7931": ["", "", "", ""], "7932": ["", "", "", ""], "7933": ["", "", "", ""], "7934": ["", "", "", ""], "7935": ["", "", "", ""], "7936": ["", "", "", ""], "7937": ["", "", "", ""], "7938": ["", "", "", ""], "7939": ["", "", "", ""], "7940": ["", "", "", ""], "7941": ["", "", "", ""], "7942": ["", "", "", ""], "7943": ["", "", "", ""], "7944": ["", "", "", ""], "7945": ["", "", "", ""], "7946": ["", "", "", ""], "7947": ["", "", "", ""], "7948": ["", "", "", ""], "7949": ["", "", "", ""], "7950": ["", "", "", ""], "7951": ["", "", "", ""], "7952": ["", "", "", ""], "7953": ["", "", "", ""], "7954": ["", "", "", ""], "7955": ["", "", "", ""], "7956": ["", "", "", ""], "7957": ["", "", "", ""], "7958": ["", "", "", ""], "7959": ["", "", "", ""], "7960": ["", "", "", ""], "7961": ["", "", "", ""], "7962": ["", "", "", ""], "7963": ["", "", "", ""], "7964": ["", "", "", ""], "7965": ["", "", "", ""], "7966": ["", "", "", ""], "7967": ["", "", "", ""], "7968": ["", "", "", ""], "7969": ["", "", "", ""], "7970": ["", "", "", ""], "7971": ["", "", "", ""], "7972": ["", "", "", ""], "7973": ["", "", "", ""], "7974": ["", "", "", ""], "7975": ["", "", "", ""], "7976": ["", "", "", ""], "7977": ["", "", "", ""], "7978": ["", "", "", ""], "7979": ["", "", "", ""], "7980": ["", "", "", ""], "7981": ["", "", "", ""], "7982": ["", "", "", ""], "7983": ["", "", "", ""], "7984": ["", "", "", ""], "7985": ["", "", "", ""], "7986": ["", "", "", ""], "7987": ["", "", "", ""], "7988": ["", "", "", ""], "7989": ["", "", "", ""], "7990": ["", "", "", ""], "7991": ["", "", "", ""], "7992": ["", "", "", ""], "7993": ["", "", "", ""], "7994": ["", "", "", ""], "7995": ["", "", "", ""], "7996": ["", "", "", ""], "7997": ["", "", "", ""], "7998": ["", "", "", ""], "7999": ["", "", "", ""], "8000": ["", "", "", ""], "8001": ["", "", "", ""], "8002": ["", "", "", ""], "8003": ["", "", "", ""], "8004": ["", "", "", ""], "8005": ["", "", "", ""], "8006": ["", "", "", ""], "8007": ["", "", "", ""], "8008": ["", "", "", ""], "8009": ["", "", "", ""], "8010": ["", "", "", ""], "8011": ["", "", "", ""], "8012": ["", "", "", ""], "8013": ["", "", "", ""], "8014": ["", "", "", ""], "8015": ["", "", "", ""], "8016": ["", "", "", ""], "8017": ["", "", "", ""], "8018": ["", "", "", ""], "8019": ["", "", "", ""], "8020": ["", "", "", ""], "8021": ["", "", "", ""], "8022": ["", "", "", ""], "8023": ["", "", "", ""], "8024": ["", "", "", ""], "8025": ["", "", "", ""], "8026": ["", "", "", ""], "8027": ["", "", "", ""], "8028": ["", "", "", ""], "8029": ["", "", "", ""], "8030": ["", "", "", ""], "8031": ["", "", "", ""], "8032": ["", "", "", ""], "8033": ["", "", "", ""], "8034": ["", "", "", ""], "8035": ["", "", "", ""], "8036": ["", "", "", ""], "8037": ["", "", "", ""], "8038": ["", "", "", ""], "8039": ["", "", "", ""], "8040": ["", "", "", ""], "8041": ["", "", "", ""], "8042": ["", "", "", ""], "8043": ["", "", "", ""], "8044": ["", "", "", ""], "8045": ["", "", "", ""], "8046": ["", "", "", ""], "8047": ["", "", "", ""], "8048": ["", "", "", ""], "8049": ["", "", "", ""], "8050": ["", "", "", ""], "8051": ["", "", "", ""], "8052": ["", "", "", ""], "8053": ["", "", "", ""], "8054": ["", "", "", ""], "8055": ["", "", "", ""], "8056": ["", "", "", ""], "8057": ["", "", "", ""], "8058": ["", "", "", ""], "8059": ["", "", "", ""], "8060": ["", "", "", ""], "8061": ["", "", "", ""], "8062": ["", "", "", ""], "8063": ["", "", "", ""], "8064": ["", "", "", ""], "8065": ["", "", "", ""], "8066": ["", "", "", ""], "8067": ["", "", "", ""], "8068": ["", "", "", ""], "8069": ["", "", "", ""], "8070": ["", "", "", ""], "8071": ["", "", "", ""], "8072": ["", "", "", ""], "8073": ["", "", "", ""], "8074": ["", "", "", ""], "8075": ["", "", "", ""], "8076": ["", "", "", ""], "8077": ["", "", "", ""], "8078": ["", "", "", ""], "8079": ["", "", "", ""], "8080": ["", "", "", ""], "8081": ["", "", "", ""], "8082": ["", "", "", ""], "8083": ["", "", "", ""], "8084": ["", "", "", ""], "8085": ["", "", "", ""], "8086": ["", "", "", ""], "8087": ["", "", "", ""], "8088": ["", "", "", ""], "8089": ["", "", "", ""], "8090": ["", "", "", ""], "8091": ["", "", "", ""], "8092": ["", "", "", ""], "8093": ["", "", "", ""], "8094": ["", "", "", ""], "8095": ["", "", "", ""], "8096": ["", "", "", ""], "8097": ["", "", "", ""], "8098": ["", "", "", ""], "8099": ["", "", "", ""], "8100": ["", "", "", ""], "8101": ["", "", "", ""], "8102": ["", "", "", ""], "8103": ["", "", "", ""], "8104": ["", "", "", ""], "8105": ["", "", "", ""], "8106": ["", "", "", ""], "8107": ["", "", "", ""], "8108": ["", "", "", ""], "8109": ["", "", "", ""], "8110": ["", "", "", ""], "8111": ["", "", "", ""], "8112": ["", "", "", ""], "8113": ["", "", "", ""], "8114": ["", "", "", ""], "8115": ["", "", "", ""], "8116": ["", "", "", ""], "8117": ["", "", "", ""], "8118": ["", "", "", ""], "8119": ["", "", "", ""], "8120": ["", "", "", ""], "8121": ["", "", "", ""], "8122": ["", "", "", ""], "8123": ["", "", "", ""], "8124": ["", "", "", ""], "8125": ["", "", "", ""], "8126": ["", "", "", ""], "8127": ["", "", "", ""], "8128": ["", "", "", ""], "8129": ["", "", "", ""], "8130": ["", "", "", ""], "8131": ["", "", "", ""], "8132": ["", "", "", ""], "8133": ["", "", "", ""], "8134": ["", "", "", ""], "8135": ["", "", "", ""], "8136": ["", "", "", ""], "8137": ["", "", "", ""], "8138": ["", "", "", ""], "8139": ["", "", "", ""], "8140": ["", "", "", ""], "8141": ["", "", "", ""], "8142": ["", "", "", ""], "8143": ["", "", "", ""], "8144": ["", "", "", ""], "8145": ["", "", "", ""], "8146": ["", "", "", ""], "8147": ["", "", "", ""], "8148": ["", "", "", ""], "8149": ["", "", "", ""], "8150": ["", "", "", ""], "8151": ["", "", "", ""], "8152": ["", "", "", ""], "8153": ["", "", "", ""], "8154": ["", "", "", ""], "8155": ["", "", "", ""], "8156": ["", "", "", ""], "8157": ["", "", "", ""], "8158": ["", "", "", ""], "8159": ["", "", "", ""], "8160": ["", "", "", ""], "8161": ["", "", "", ""], "8162": ["", "", "", ""], "8163": ["", "", "", ""], "8164": ["", "", "", ""], "8165": ["", "", "", ""], "8166": ["", "", "", ""], "8167": ["", "", "", ""], "8168": ["", "", "", ""], "8169": ["", "", "", ""], "8170": ["", "", "", ""], "8171": ["", "", "", ""], "8172": ["", "", "", ""], "8173": ["", "", "", ""], "8174": ["", "", "", ""], "8175": ["", "", "", ""], "8176": ["", "", "", ""], "8177": ["", "", "", ""], "8178": ["", "", "", ""], "8179": ["", "", "", ""], "8180": ["", "", "", ""], "8181": ["", "", "", ""], "8182": ["", "", "", ""], "8183": ["", "", "", ""], "8184": ["", "", "", ""], "8185": ["", "", "", ""], "8186": ["", "", "", ""], "8187": ["", "", "", ""], "8188": ["", "", "", ""], "8189": ["", "", "", ""], "8190": ["", "", "", ""], "8191": ["", "", "", ""], "8192": ["", "", "", ""], "8193": ["", "", "", ""], "8194": ["", "", "", ""], "8195": ["", "", "", ""], "8196": ["", "", "", ""], "8197": ["", "", "", ""], "8198": ["", "", "", ""], "8199": ["", "", "", ""], "8200": ["", "", "", ""], "8201": ["", "", "", ""], "8202": ["", "", "", ""], "8203": ["", "", "", ""], "8204": ["", "", "", ""], "8205": ["", "", "", ""], "8206": ["", "", "", ""], "8207": ["", "", "", ""], "8208": ["", "", "", ""], "8209": ["", "", "", ""], "8210": ["", "", "", ""], "8211": ["", "", "", ""], "8212": ["", "", "", ""], "8213": ["", "", "", ""], "8214": ["", "", "", ""], "8215": ["", "", "", ""], "8216": ["", "", "", ""], "8217": ["", "", "", ""], "8218": ["", "", "", ""], "8219": ["", "", "", ""], "8220": ["", "", "", ""], "8221": ["", "", "", ""], "8222": ["", "", "", ""], "8223": ["", "", "", ""], "8224": ["", "", "", ""], "8225": ["", "", "", ""], "8226": ["", "", "", ""], "8227": ["", "", "", ""], "8228": ["", "", "", ""], "8229": ["", "", "", ""], "8230": ["", "", "", ""], "8231": ["", "", "", ""], "8232": ["", "", "", ""], "8233": ["", "", "", ""], "8234": ["", "", "", ""], "8235": ["", "", "", ""], "8236": ["", "", "", ""], "8237": ["", "", "", ""], "8238": ["", "", "", ""], "8239": ["", "", "", ""], "8240": ["", "", "", ""], "8241": ["", "", "", ""], "8242": ["", "", "", ""], "8243": ["", "", "", ""], "8244": ["", "", "", ""], "8245": ["", "", "", ""], "8246": ["", "", "", ""], "8247": ["", "", "", ""], "8248": ["", "", "", ""], "8249": ["", "", "", ""], "8250": ["", "", "", ""], "8251": ["", "", "", ""], "8252": ["", "", "", ""], "8253": ["", "", "", ""], "8254": ["", "", "", ""], "8255": ["", "", "", ""], "8256": ["", "", "", ""], "8257": ["", "", "", ""], "8258": ["", "", "", ""], "8259": ["", "", "", ""], "8260": ["", "", "", ""], "8261": ["", "", "", ""], "8262": ["", "", "", ""], "8263": ["", "", "", ""], "8264": ["", "", "", ""], "8265": ["", "", "", ""], "8266": ["", "", "", ""], "8267": ["", "", "", ""], "8268": ["", "", "", ""], "8269": ["", "", "", ""], "8270": ["", "", "", ""], "8271": ["", "", "", ""], "8272": ["", "", "", ""], "8273": ["", "", "", ""], "8274": ["", "", "", ""], "8275": ["", "", "", ""], "8276": ["", "", "", ""], "8277": ["", "", "", ""], "8278": ["", "", "", ""], "8279": ["", "", "", ""], "8280": ["", "", "", ""], "8281": ["", "", "", ""], "8282": ["", "", "", ""], "8283": ["", "", "", ""], "8284": ["", "", "", ""], "8285": ["", "", "", ""], "8286": ["", "", "", ""], "8287": ["", "", "", ""], "8288": ["", "", "", ""], "8289": ["", "", "", ""], "8290": ["", "", "", ""], "8291": ["", "", "", ""], "8292": ["", "", "", ""], "8293": ["", "", "", ""], "8294": ["", "", "", ""], "8295": ["", "", "", ""], "8296": ["", "", "", ""], "8297": ["", "", "", ""], "8298": ["", "", "", ""], "8299": ["", "", "", ""], "8300": ["", "", "", ""], "8301": ["", "", "", ""], "8302": ["", "", "", ""], "8303": ["", "", "", ""], "8304": ["", "", "", ""], "8305": ["", "", "", ""], "8306": ["", "", "", ""], "8307": ["", "", "", ""], "8308": ["", "", "", ""], "8309": ["", "", "", ""], "8310": ["", "", "", ""], "8311": ["", "", "", ""], "8312": ["", "", "", ""], "8313": ["", "", "", ""], "8314": ["", "", "", ""], "8315": ["", "", "", ""], "8316": ["", "", "", ""], "8317": ["", "", "", ""], "8318": ["", "", "", ""], "8319": ["", "", "", ""], "8320": ["", "", "", ""], "8321": ["", "", "", ""], "8322": ["", "", "", ""], "8323": ["", "", "", ""], "8324": ["", "", "", ""], "8325": ["", "", "", ""], "8326": ["", "", "", ""], "8327": ["", "", "", ""], "8328": ["", "", "", ""], "8329": ["", "", "", ""], "8330": ["", "", "", ""], "8331": ["", "", "", ""], "8332": ["", "", "", ""], "8333": ["", "", "", ""], "8334": ["", "", "", ""], "8335": ["", "", "", ""], "8336": ["", "", "", ""], "8337": ["", "", "", ""], "8338": ["", "", "", ""], "8339": ["", "", "", ""], "8340": ["", "", "", ""], "8341": ["", "", "", ""], "8342": ["", "", "", ""], "8343": ["", "", "", ""], "8344": ["", "", "", ""], "8345": ["", "", "", ""], "8346": ["", "", "", ""], "8347": ["", "", "", ""], "8348": ["", "", "", ""], "8349": ["", "", "", ""], "8350": ["", "", "", ""], "8351": ["", "", "", ""], "8352": ["", "", "", ""], "8353": ["", "", "", ""], "8354": ["", "", "", ""], "8355": ["", "", "", ""], "8356": ["", "", "", ""], "8357": ["", "", "", ""], "8358": ["", "", "", ""], "8359": ["", "", "", ""], "8360": ["", "", "", ""], "8361": ["", "", "", ""], "8362": ["", "", "", ""], "8363": ["", "", "", ""], "8364": ["", "", "", ""], "8365": ["", "", "", ""], "8366": ["", "", "", ""], "8367": ["", "", "", ""], "8368": ["", "", "", ""], "8369": ["", "", "", ""], "8370": ["", "", "", ""], "8371": ["", "", "", ""], "8372": ["", "", "", ""], "8373": ["", "", "", ""], "8374": ["", "", "", ""], "8375": ["", "", "", ""], "8376": ["", "", "", ""], "8377": ["", "", "", ""], "8378": ["", "", "", ""], "8379": ["", "", "", ""], "8380": ["", "", "", ""], "8381": ["", "", "", ""], "8382": ["", "", "", ""], "8383": ["", "", "", ""], "8384": ["", "", "", ""], "8385": ["", "", "", ""], "8386": ["", "", "", ""], "8387": ["", "", "", ""], "8388": ["", "", "", ""], "8389": ["", "", "", ""], "8390": ["", "", "", ""], "8391": ["", "", "", ""], "8392": ["", "", "", ""], "8393": ["", "", "", ""], "8394": ["", "", "", ""], "8395": ["", "", "", ""], "8396": ["", "", "", ""], "8397": ["", "", "", ""], "8398": ["", "", "", ""], "8399": ["", "", "", ""], "8400": ["", "", "", ""], "8401": ["", "", "", ""], "8402": ["", "", "", ""], "8403": ["", "", "", ""], "8404": ["", "", "", ""], "8405": ["", "", "", ""], "8406": ["", "", "", ""], "8407": ["", "", "", ""], "8408": ["", "", "", ""], "8409": ["", "", "", ""], "8410": ["", "", "", ""], "8411": ["", "", "", ""], "8412": ["", "", "", ""], "8413": ["", "", "", ""], "8414": ["", "", "", ""], "8415": ["", "", "", ""], "8416": ["", "", "", ""], "8417": ["", "", "", ""], "8418": ["", "", "", ""], "8419": ["", "", "", ""], "8420": ["", "", "", ""], "8421": ["", "", "", ""], "8422": ["", "", "", ""], "8423": ["", "", "", ""], "8424": ["", "", "", ""], "8425": ["", "", "", ""], "8426": ["", "", "", ""], "8427": ["", "", "", ""], "8428": ["", "", "", ""], "8429": ["", "", "", ""], "8430": ["", "", "", ""], "8431": ["", "", "", ""], "8432": ["", "", "", ""], "8433": ["", "", "", ""], "8434": ["", "", "", ""], "8435": ["", "", "", ""], "8436": ["", "", "", ""], "8437": ["", "", "", ""], "8438": ["", "", "", ""], "8439": ["", "", "", ""], "8440": ["", "", "", ""], "8441": ["", "", "", ""], "8442": ["", "", "", ""], "8443": ["", "", "", ""], "8444": ["", "", "", ""], "8445": ["", "", "", ""], "8446": ["", "", "", ""], "8447": ["", "", "", ""], "8448": ["", "", "", ""], "8449": ["", "", "", ""], "8450": ["", "", "", ""], "8451": ["", "", "", ""], "8452": ["", "", "", ""], "8453": ["", "", "", ""], "8454": ["", "", "", ""], "8455": ["", "", "", ""], "8456": ["", "", "", ""], "8457": ["", "", "", ""], "8458": ["", "", "", ""], "8459": ["", "", "", ""], "8460": ["", "", "", ""], "8461": ["", "", "", ""], "8462": ["", "", "", ""], "8463": ["", "", "", ""], "8464": ["", "", "", ""], "8465": ["", "", "", ""], "8466": ["", "", "", ""], "8467": ["", "", "", ""], "8468": ["", "", "", ""], "8469": ["", "", "", ""], "8470": ["", "", "", ""], "8471": ["", "", "", ""], "8472": ["", "", "", ""], "8473": ["", "", "", ""], "8474": ["", "", "", ""], "8475": ["", "", "", ""], "8476": ["", "", "", ""], "8477": ["", "", "", ""], "8478": ["", "", "", ""], "8479": ["", "", "", ""], "8480": ["", "", "", ""], "8481": ["", "", "", ""], "8482": ["", "", "", ""], "8483": ["", "", "", ""], "8484": ["", "", "", ""], "8485": ["", "", "", ""], "8486": ["", "", "", ""], "8487": ["", "", "", ""], "8488": ["", "", "", ""], "8489": ["", "", "", ""], "8490": ["", "", "", ""], "8491": ["", "", "", ""], "8492": ["", "", "", ""], "8493": ["", "", "", ""], "8494": ["", "", "", ""], "8495": ["", "", "", ""], "8496": ["", "", "", ""], "8497": ["", "", "", ""], "8498": ["", "", "", ""], "8499": ["", "", "", ""], "8500": ["", "", "", ""], "8501": ["", "", "", ""], "8502": ["", "", "", ""], "8503": ["", "", "", ""], "8504": ["", "", "", ""], "8505": ["", "", "", ""], "8506": ["", "", "", ""], "8507": ["", "", "", ""], "8508": ["", "", "", ""], "8509": ["", "", "", ""], "8510": ["", "", "", ""], "8511": ["", "", "", ""], "8512": ["", "", "", ""], "8513": ["", "", "", ""], "8514": ["", "", "", ""], "8515": ["", "", "", ""], "8516": ["", "", "", ""], "8517": ["", "", "", ""], "8518": ["", "", "", ""], "8519": ["", "", "", ""], "8520": ["", "", "", ""], "8521": ["", "", "", ""], "8522": ["", "", "", ""], "8523": ["", "", "", ""], "8524": ["", "", "", ""], "8525": ["", "", "", ""], "8526": ["", "", "", ""], "8527": ["", "", "", ""], "8528": ["", "", "", ""], "8529": ["", "", "", ""], "8530": ["", "", "", ""], "8531": ["", "", "", ""], "8532": ["", "", "", ""], "8533": ["", "", "", ""], "8534": ["", "", "", ""], "8535": ["", "", "", ""], "8536": ["", "", "", ""], "8537": ["", "", "", ""], "8538": ["", "", "", ""], "8539": ["", "", "", ""], "8540": ["", "", "", ""], "8541": ["", "", "", ""], "8542": ["", "", "", ""], "8543": ["", "", "", ""], "8544": ["", "", "", ""], "8545": ["", "", "", ""], "8546": ["", "", "", ""], "8547": ["", "", "", ""], "8548": ["", "", "", ""], "8549": ["", "", "", ""], "8550": ["", "", "", ""], "8551": ["", "", "", ""], "8552": ["", "", "", ""], "8553": ["", "", "", ""], "8554": ["", "", "", ""], "8555": ["", "", "", ""], "8556": ["", "", "", ""], "8557": ["", "", "", ""], "8558": ["", "", "", ""], "8559": ["", "", "", ""], "8560": ["", "", "", ""], "8561": ["", "", "", ""], "8562": ["", "", "", ""], "8563": ["", "", "", ""], "8564": ["", "", "", ""], "8565": ["", "", "", ""], "8566": ["", "", "", ""], "8567": ["", "", "", ""], "8568": ["", "", "", ""], "8569": ["", "", "", ""], "8570": ["", "", "", ""], "8571": ["", "", "", ""], "8572": ["", "", "", ""], "8573": ["", "", "", ""], "8574": ["", "", "", ""], "8575": ["", "", "", ""], "8576": ["", "", "", ""], "8577": ["", "", "", ""], "8578": ["", "", "", ""], "8579": ["", "", "", ""], "8580": ["", "", "", ""], "8581": ["", "", "", ""], "8582": ["", "", "", ""], "8583": ["", "", "", ""], "8584": ["", "", "", ""], "8585": ["", "", "", ""], "8586": ["", "", "", ""], "8587": ["", "", "", ""], "8588": ["", "", "", ""], "8589": ["", "", "", ""], "8590": ["", "", "", ""], "8591": ["", "", "", ""], "8592": ["", "", "", ""], "8593": ["", "", "", ""], "8594": ["", "", "", ""], "8595": ["", "", "", ""], "8596": ["", "", "", ""], "8597": ["", "", "", ""], "8598": ["", "", "", ""], "8599": ["", "", "", ""], "8600": ["", "", "", ""], "8601": ["", "", "", ""], "8602": ["", "", "", ""], "8603": ["", "", "", ""], "8604": ["", "", "", ""], "8605": ["", "", "", ""], "8606": ["", "", "", ""], "8607": ["", "", "", ""], "8608": ["", "", "", ""], "8609": ["", "", "", ""], "8610": ["", "", "", ""], "8611": ["", "", "", ""], "8612": ["", "", "", ""], "8613": ["", "", "", ""], "8614": ["", "", "", ""], "8615": ["", "", "", ""], "8616": ["", "", "", ""], "8617": ["", "", "", ""], "8618": ["", "", "", ""], "8619": ["", "", "", ""], "8620": ["", "", "", ""], "8621": ["", "", "", ""], "8622": ["", "", "", ""], "8623": ["", "", "", ""], "8624": ["", "", "", ""], "8625": ["", "", "", ""], "8626": ["", "", "", ""], "8627": ["", "", "", ""], "8628": ["", "", "", ""], "8629": ["", "", "", ""], "8630": ["", "", "", ""], "8631": ["", "", "", ""], "8632": ["", "", "", ""], "8633": ["", "", "", ""], "8634": ["", "", "", ""], "8635": ["", "", "", ""], "8636": ["", "", "", ""], "8637": ["", "", "", ""], "8638": ["", "", "", ""], "8639": ["", "", "", ""], "8640": ["", "", "", ""], "8641": ["", "", "", ""], "8642": ["", "", "", ""], "8643": ["", "", "", ""], "8644": ["", "", "", ""], "8645": ["", "", "", ""], "8646": ["", "", "", ""], "8647": ["", "", "", ""], "8648": ["", "", "", ""], "8649": ["", "", "", ""], "8650": ["", "", "", ""], "8651": ["", "", "", ""], "8652": ["", "", "", ""], "8653": ["", "", "", ""], "8654": ["", "", "", ""], "8655": ["", "", "", ""], "8656": ["", "", "", ""], "8657": ["", "", "", ""], "8658": ["", "", "", ""], "8659": ["", "", "", ""], "8660": ["", "", "", ""], "8661": ["", "", "", ""], "8662": ["", "", "", ""], "8663": ["", "", "", ""], "8664": ["", "", "", ""], "8665": ["", "", "", ""], "8666": ["", "", "", ""], "8667": ["", "", "", ""], "8668": ["", "", "", ""], "8669": ["", "", "", ""], "8670": ["", "", "", ""], "8671": ["", "", "", ""], "8672": ["", "", "", ""], "8673": ["", "", "", ""], "8674": ["", "", "", ""], "8675": ["", "", "", ""], "8676": ["", "", "", ""], "8677": ["", "", "", ""], "8678": ["", "", "", ""], "8679": ["", "", "", ""], "8680": ["", "", "", ""], "8681": ["", "", "", ""], "8682": ["", "", "", ""], "8683": ["", "", "", ""], "8684": ["", "", "", ""], "8685": ["", "", "", ""], "8686": ["", "", "", ""], "8687": ["", "", "", ""], "8688": ["", "", "", ""], "8689": ["", "", "", ""], "8690": ["", "", "", ""], "8691": ["", "", "", ""], "8692": ["", "", "", ""], "8693": ["", "", "", ""], "8694": ["", "", "", ""], "8695": ["", "", "", ""], "8696": ["", "", "", ""], "8697": ["", "", "", ""], "8698": ["", "", "", ""], "8699": ["", "", "", ""], "8700": ["", "", "", ""], "8701": ["", "", "", ""], "8702": ["", "", "", ""], "8703": ["", "", "", ""], "8704": ["", "", "", ""], "8705": ["", "", "", ""], "8706": ["", "", "", ""], "8707": ["", "", "", ""], "8708": ["", "", "", ""], "8709": ["", "", "", ""], "8710": ["", "", "", ""], "8711": ["", "", "", ""], "8712": ["", "", "", ""], "8713": ["", "", "", ""], "8714": ["", "", "", ""], "8715": ["", "", "", ""], "8716": ["", "", "", ""], "8717": ["", "", "", ""], "8718": ["", "", "", ""], "8719": ["", "", "", ""], "8720": ["", "", "", ""], "8721": ["", "", "", ""], "8722": ["", "", "", ""], "8723": ["", "", "", ""], "8724": ["", "", "", ""], "8725": ["", "", "", ""], "8726": ["", "", "", ""], "8727": ["", "", "", ""], "8728": ["", "", "", ""], "8729": ["", "", "", ""], "8730": ["", "", "", ""], "8731": ["", "", "", ""], "8732": ["", "", "", ""], "8733": ["", "", "", ""], "8734": ["", "", "", ""], "8735": ["", "", "", ""], "8736": ["", "", "", ""], "8737": ["", "", "", ""], "8738": ["", "", "", ""], "8739": ["", "", "", ""], "8740": ["", "", "", ""], "8741": ["", "", "", ""], "8742": ["", "", "", ""], "8743": ["", "", "", ""], "8744": ["", "", "", ""], "8745": ["", "", "", ""], "8746": ["", "", "", ""], "8747": ["", "", "", ""], "8748": ["", "", "", ""], "8749": ["", "", "", ""], "8750": ["", "", "", ""], "8751": ["", "", "", ""], "8752": ["", "", "", ""], "8753": ["", "", "", ""], "8754": ["", "", "", ""], "8755": ["", "", "", ""], "8756": ["", "", "", ""], "8757": ["", "", "", ""], "8758": ["", "", "", ""], "8759": ["", "", "", ""], "8760": ["", "", "", ""], "8761": ["", "", "", ""], "8762": ["", "", "", ""], "8763": ["", "", "", ""], "8764": ["", "", "", ""], "8765": ["", "", "", ""], "8766": ["", "", "", ""], "8767": ["", "", "", ""], "8768": ["", "", "", ""], "8769": ["", "", "", ""], "8770": ["", "", "", ""], "8771": ["", "", "", ""], "8772": ["", "", "", ""], "8773": ["", "", "", ""], "8774": ["", "", "", ""], "8775": ["", "", "", ""], "8776": ["", "", "", ""], "8777": ["", "", "", ""], "8778": ["", "", "", ""], "8779": ["", "", "", ""], "8780": ["", "", "", ""], "8781": ["", "", "", ""], "8782": ["", "", "", ""], "8783": ["", "", "", ""], "8784": ["", "", "", ""], "8785": ["", "", "", ""], "8786": ["", "", "", ""], "8787": ["", "", "", ""], "8788": ["", "", "", ""], "8789": ["", "", "", ""], "8790": ["", "", "", ""], "8791": ["", "", "", ""], "8792": ["", "", "", ""], "8793": ["", "", "", ""], "8794": ["", "", "", ""], "8795": ["", "", "", ""], "8796": ["", "", "", ""], "8797": ["", "", "", ""], "8798": ["", "", "", ""], "8799": ["", "", "", ""], "8800": ["", "", "", ""], "8801": ["", "", "", ""], "8802": ["", "", "", ""], "8803": ["", "", "", ""], "8804": ["", "", "", ""], "8805": ["", "", "", ""], "8806": ["", "", "", ""], "8807": ["", "", "", ""], "8808": ["", "", "", ""], "8809": ["", "", "", ""], "8810": ["", "", "", ""], "8811": ["", "", "", ""], "8812": ["", "", "", ""], "8813": ["", "", "", ""], "8814": ["", "", "", ""], "8815": ["", "", "", ""], "8816": ["", "", "", ""], "8817": ["", "", "", ""], "8818": ["", "", "", ""], "8819": ["", "", "", ""], "8820": ["", "", "", ""], "8821": ["", "", "", ""], "8822": ["", "", "", ""], "8823": ["", "", "", ""], "8824": ["", "", "", ""], "8825": ["", "", "", ""], "8826": ["", "", "", ""], "8827": ["", "", "", ""], "8828": ["", "", "", ""], "8829": ["", "", "", ""], "8830": ["", "", "", ""], "8831": ["", "", "", ""], "8832": ["", "", "", ""], "8833": ["", "", "", ""], "8834": ["", "", "", ""], "8835": ["", "", "", ""], "8836": ["", "", "", ""], "8837": ["", "", "", ""], "8838": ["", "", "", ""], "8839": ["", "", "", ""], "8840": ["", "", "", ""], "8841": ["", "", "", ""], "8842": ["", "", "", ""], "8843": ["", "", "", ""], "8844": ["", "", "", ""], "8845": ["", "", "", ""], "8846": ["", "", "", ""], "8847": ["", "", "", ""], "8848": ["", "", "", ""], "8849": ["", "", "", ""], "8850": ["", "", "", ""], "8851": ["", "", "", ""], "8852": ["", "", "", ""], "8853": ["", "", "", ""], "8854": ["", "", "", ""], "8855": ["", "", "", ""], "8856": ["", "", "", ""], "8857": ["", "", "", ""], "8858": ["", "", "", ""], "8859": ["", "", "", ""], "8860": ["", "", "", ""], "8861": ["", "", "", ""], "8862": ["", "", "", ""], "8863": ["", "", "", ""], "8864": ["", "", "", ""], "8865": ["", "", "", ""], "8866": ["", "", "", ""], "8867": ["", "", "", ""], "8868": ["", "", "", ""], "8869": ["", "", "", ""], "8870": ["", "", "", ""], "8871": ["", "", "", ""], "8872": ["", "", "", ""], "8873": ["", "", "", ""], "8874": ["", "", "", ""], "8875": ["", "", "", ""], "8876": ["", "", "", ""], "8877": ["", "", "", ""], "8878": ["", "", "", ""], "8879": ["", "", "", ""], "8880": ["", "", "", ""], "8881": ["", "", "", ""], "8882": ["", "", "", ""], "8883": ["", "", "", ""], "8884": ["", "", "", ""], "8885": ["", "", "", ""], "8886": ["", "", "", ""], "8887": ["", "", "", ""], "8888": ["", "", "", ""], "8889": ["", "", "", ""], "8890": ["", "", "", ""], "8891": ["", "", "", ""], "8892": ["", "", "", ""], "8893": ["", "", "", ""], "8894": ["", "", "", ""], "8895": ["", "", "", ""], "8896": ["", "", "", ""], "8897": ["", "", "", ""], "8898": ["", "", "", ""], "8899": ["", "", "", ""], "8900": ["", "", "", ""], "8901": ["", "", "", ""], "8902": ["", "", "", ""], "8903": ["", "", "", ""], "8904": ["", "", "", ""], "8905": ["", "", "", ""], "8906": ["", "", "", ""], "8907": ["", "", "", ""], "8908": ["", "", "", ""], "8909": ["", "", "", ""], "8910": ["", "", "", ""], "8911": ["", "", "", ""], "8912": ["", "", "", ""], "8913": ["", "", "", ""], "8914": ["", "", "", ""], "8915": ["", "", "", ""], "8916": ["", "", "", ""], "8917": ["", "", "", ""], "8918": ["", "", "", ""], "8919": ["", "", "", ""], "8920": ["", "", "", ""], "8921": ["", "", "", ""], "8922": ["", "", "", ""], "8923": ["", "", "", ""], "8924": ["", "", "", ""], "8925": ["", "", "", ""], "8926": ["", "", "", ""], "8927": ["", "", "", ""], "8928": ["", "", "", ""], "8929": ["", "", "", ""], "8930": ["", "", "", ""], "8931": ["", "", "", ""], "8932": ["", "", "", ""], "8933": ["", "", "", ""], "8934": ["", "", "", ""], "8935": ["", "", "", ""], "8936": ["", "", "", ""], "8937": ["", "", "", ""], "8938": ["", "", "", ""], "8939": ["", "", "", ""], "8940": ["", "", "", ""], "8941": ["", "", "", ""], "8942": ["", "", "", ""], "8943": ["", "", "", ""], "8944": ["", "", "", ""], "8945": ["", "", "", ""], "8946": ["", "", "", ""], "8947": ["", "", "", ""], "8948": ["", "", "", ""], "8949": ["", "", "", ""], "8950": ["", "", "", ""], "8951": ["", "", "", ""], "8952": ["", "", "", ""], "8953": ["", "", "", ""], "8954": ["", "", "", ""], "8955": ["", "", "", ""], "8956": ["", "", "", ""], "8957": ["", "", "", ""], "8958": ["", "", "", ""], "8959": ["", "", "", ""], "8960": ["", "", "", ""], "8961": ["", "", "", ""], "8962": ["", "", "", ""], "8963": ["", "", "", ""], "8964": ["", "", "", ""], "8965": ["", "", "", ""], "8966": ["", "", "", ""], "8967": ["", "", "", ""], "8968": ["", "", "", ""], "8969": ["", "", "", ""], "8970": ["", "", "", ""], "8971": ["", "", "", ""], "8972": ["", "", "", ""], "8973": ["", "", "", ""], "8974": ["", "", "", ""], "8975": ["", "", "", ""], "8976": ["", "", "", ""], "8977": ["", "", "", ""], "8978": ["", "", "", ""], "8979": ["", "", "", ""], "8980": ["", "", "", ""], "8981": ["", "", "", ""], "8982": ["", "", "", ""], "8983": ["", "", "", ""], "8984": ["", "", "", ""], "8985": ["", "", "", ""], "8986": ["", "", "", ""], "8987": ["", "", "", ""], "8988": ["", "", "", ""], "8989": ["", "", "", ""], "8990": ["", "", "", ""], "8991": ["", "", "", ""], "8992": ["", "", "", ""], "8993": ["", "", "", ""], "8994": ["", "", "", ""], "8995": ["", "", "", ""], "8996": ["", "", "", ""], "8997": ["", "", "", ""], "8998": ["", "", "", ""], "8999": ["", "", "", ""], "9000": ["", "", "", ""], "9001": ["", "", "", ""], "9002": ["", "", "", ""], "9003": ["", "", "", ""], "9004": ["", "", "", ""], "9005": ["", "", "", ""], "9006": ["", "", "", ""], "9007": ["", "", "", ""], "9008": ["", "", "", ""], "9009": ["", "", "", ""], "9010": ["", "", "", ""], "9011": ["", "", "", ""], "9012": ["", "", "", ""], "9013": ["", "", "", ""], "9014": ["", "", "", ""], "9015": ["", "", "", ""], "9016": ["", "", "", ""], "9017": ["", "", "", ""], "9018": ["", "", "", ""], "9019": ["", "", "", ""], "9020": ["", "", "", ""], "9021": ["", "", "", ""], "9022": ["", "", "", ""], "9023": ["", "", "", ""], "9024": ["", "", "", ""], "9025": ["", "", "", ""], "9026": ["", "", "", ""], "9027": ["", "", "", ""], "9028": ["", "", "", ""], "9029": ["", "", "", ""], "9030": ["", "", "", ""], "9031": ["", "", "", ""], "9032": ["", "", "", ""], "9033": ["", "", "", ""], "9034": ["", "", "", ""], "9035": ["", "", "", ""], "9036": ["", "", "", ""], "9037": ["", "", "", ""], "9038": ["", "", "", ""], "9039": ["", "", "", ""], "9040": ["", "", "", ""], "9041": ["", "", "", ""], "9042": ["", "", "", ""], "9043": ["", "", "", ""], "9044": ["", "", "", ""], "9045": ["", "", "", ""], "9046": ["", "", "", ""], "9047": ["", "", "", ""], "9048": ["", "", "", ""], "9049": ["", "", "", ""], "9050": ["", "", "", ""], "9051": ["", "", "", ""], "9052": ["", "", "", ""], "9053": ["", "", "", ""], "9054": ["", "", "", ""], "9055": ["", "", "", ""], "9056": ["", "", "", ""], "9057": ["", "", "", ""], "9058": ["", "", "", ""], "9059": ["", "", "", ""], "9060": ["", "", "", ""], "9061": ["", "", "", ""], "9062": ["", "", "", ""], "9063": ["", "", "", ""], "9064": ["", "", "", ""], "9065": ["", "", "", ""], "9066": ["", "", "", ""], "9067": ["", "", "", ""], "9068": ["", "", "", ""], "9069": ["", "", "", ""], "9070": ["", "", "", ""], "9071": ["", "", "", ""], "9072": ["", "", "", ""], "9073": ["", "", "", ""], "9074": ["", "", "", ""], "9075": ["", "", "", ""], "9076": ["", "", "", ""], "9077": ["", "", "", ""], "9078": ["", "", "", ""], "9079": ["", "", "", ""], "9080": ["", "", "", ""], "9081": ["", "", "", ""], "9082": ["", "", "", ""], "9083": ["", "", "", ""], "9084": ["", "", "", ""], "9085": ["", "", "", ""], "9086": ["", "", "", ""], "9087": ["", "", "", ""], "9088": ["", "", "", ""], "9089": ["", "", "", ""], "9090": ["", "", "", ""], "9091": ["", "", "", ""], "9092": ["", "", "", ""], "9093": ["", "", "", ""], "9094": ["", "", "", ""], "9095": ["", "", "", ""], "9096": ["", "", "", ""], "9097": ["", "", "", ""], "9098": ["", "", "", ""], "9099": ["", "", "", ""], "9100": ["", "", "", ""], "9101": ["", "", "", ""], "9102": ["", "", "", ""], "9103": ["", "", "", ""], "9104": ["", "", "", ""], "9105": ["", "", "", ""], "9106": ["", "", "", ""], "9107": ["", "", "", ""], "9108": ["", "", "", ""], "9109": ["", "", "", ""], "9110": ["", "", "", ""], "9111": ["", "", "", ""], "9112": ["", "", "", ""], "9113": ["", "", "", ""], "9114": ["", "", "", ""], "9115": ["", "", "", ""], "9116": ["", "", "", ""], "9117": ["", "", "", ""], "9118": ["", "", "", ""], "9119": ["", "", "", ""], "9120": ["", "", "", ""], "9121": ["", "", "", ""], "9122": ["", "", "", ""], "9123": ["", "", "", ""], "9124": ["", "", "", ""], "9125": ["", "", "", ""], "9126": ["", "", "", ""], "9127": ["", "", "", ""], "9128": ["", "", "", ""], "9129": ["", "", "", ""], "9130": ["", "", "", ""], "9131": ["", "", "", ""], "9132": ["", "", "", ""], "9133": ["", "", "", ""], "9134": ["", "", "", ""], "9135": ["", "", "", ""], "9136": ["", "", "", ""], "9137": ["", "", "", ""], "9138": ["", "", "", ""], "9139": ["", "", "", ""], "9140": ["", "", "", ""], "9141": ["", "", "", ""], "9142": ["", "", "", ""], "9143": ["", "", "", ""], "9144": ["", "", "", ""], "9145": ["", "", "", ""], "9146": ["", "", "", ""], "9147": ["", "", "", ""], "9148": ["", "", "", ""], "9149": ["", "", "", ""], "9150": ["", "", "", ""], "9151": ["", "", "", ""], "9152": ["", "", "", ""], "9153": ["", "", "", ""], "9154": ["", "", "", ""], "9155": ["", "", "", ""], "9156": ["", "", "", ""], "9157": ["", "", "", ""], "9158": ["", "", "", ""], "9159": ["", "", "", ""], "9160": ["", "", "", ""], "9161": ["", "", "", ""], "9162": ["", "", "", ""], "9163": ["", "", "", ""], "9164": ["", "", "", ""], "9165": ["", "", "", ""], "9166": ["", "", "", ""], "9167": ["", "", "", ""], "9168": ["", "", "", ""], "9169": ["", "", "", ""], "9170": ["", "", "", ""], "9171": ["", "", "", ""], "9172": ["", "", "", ""], "9173": ["", "", "", ""], "9174": ["", "", "", ""], "9175": ["", "", "", ""], "9176": ["", "", "", ""], "9177": ["", "", "", ""], "9178": ["", "", "", ""], "9179": ["", "", "", ""], "9180": ["", "", "", ""], "9181": ["", "", "", ""], "9182": ["", "", "", ""], "9183": ["", "", "", ""], "9184": ["", "", "", ""], "9185": ["", "", "", ""], "9186": ["", "", "", ""], "9187": ["", "", "", ""], "9188": ["", "", "", ""], "9189": ["", "", "", ""], "9190": ["", "", "", ""], "9191": ["", "", "", ""], "9192": ["", "", "", ""], "9193": ["", "", "", ""], "9194": ["", "", "", ""], "9195": ["", "", "", ""], "9196": ["", "", "", ""], "9197": ["", "", "", ""], "9198": ["", "", "", ""], "9199": ["", "", "", ""], "9200": ["", "", "", ""], "9201": ["", "", "", ""], "9202": ["", "", "", ""], "9203": ["", "", "", ""], "9204": ["", "", "", ""], "9205": ["", "", "", ""], "9206": ["", "", "", ""], "9207": ["", "", "", ""], "9208": ["", "", "", ""], "9209": ["", "", "", ""], "9210": ["", "", "", ""], "9211": ["", "", "", ""], "9212": ["", "", "", ""], "9213": ["", "", "", ""], "9214": ["", "", "", ""], "9215": ["", "", "", ""], "9216": ["", "", "", ""], "9217": ["", "", "", ""], "9218": ["", "", "", ""], "9219": ["", "", "", ""], "9220": ["", "", "", ""], "9221": ["", "", "", ""], "9222": ["", "", "", ""], "9223": ["", "", "", ""], "9224": ["", "", "", ""], "9225": ["", "", "", ""], "9226": ["", "", "", ""], "9227": ["", "", "", ""], "9228": ["", "", "", ""], "9229": ["", "", "", ""], "9230": ["", "", "", ""], "9231": ["", "", "", ""], "9232": ["", "", "", ""], "9233": ["", "", "", ""], "9234": ["", "", "", ""], "9235": ["", "", "", ""], "9236": ["", "", "", ""], "9237": ["", "", "", ""], "9238": ["", "", "", ""], "9239": ["", "", "", ""], "9240": ["", "", "", ""], "9241": ["", "", "", ""], "9242": ["", "", "", ""], "9243": ["", "", "", ""], "9244": ["", "", "", ""], "9245": ["", "", "", ""], "9246": ["", "", "", ""], "9247": ["", "", "", ""], "9248": ["", "", "", ""], "9249": ["", "", "", ""], "9250": ["", "", "", ""], "9251": ["", "", "", ""], "9252": ["", "", "", ""], "9253": ["", "", "", ""], "9254": ["", "", "", ""], "9255": ["", "", "", ""], "9256": ["", "", "", ""], "9257": ["", "", "", ""], "9258": ["", "", "", ""], "9259": ["", "", "", ""], "9260": ["", "", "", ""], "9261": ["", "", "", ""], "9262": ["", "", "", ""], "9263": ["", "", "", ""], "9264": ["", "", "", ""], "9265": ["", "", "", ""], "9266": ["", "", "", ""], "9267": ["", "", "", ""], "9268": ["", "", "", ""], "9269": ["", "", "", ""], "9270": ["", "", "", ""], "9271": ["", "", "", ""], "9272": ["", "", "", ""], "9273": ["", "", "", ""], "9274": ["", "", "", ""], "9275": ["", "", "", ""], "9276": ["", "", "", ""], "9277": ["", "", "", ""], "9278": ["", "", "", ""], "9279": ["", "", "", ""], "9280": ["", "", "", ""], "9281": ["", "", "", ""], "9282": ["", "", "", ""], "9283": ["", "", "", ""], "9284": ["", "", "", ""], "9285": ["", "", "", ""], "9286": ["", "", "", ""], "9287": ["", "", "", ""], "9288": ["", "", "", ""], "9289": ["", "", "", ""], "9290": ["", "", "", ""], "9291": ["", "", "", ""], "9292": ["", "", "", ""], "9293": ["", "", "", ""], "9294": ["", "", "", ""], "9295": ["", "", "", ""], "9296": ["", "", "", ""], "9297": ["", "", "", ""], "9298": ["", "", "", ""], "9299": ["", "", "", ""], "9300": ["", "", "", ""], "9301": ["", "", "", ""], "9302": ["", "", "", ""], "9303": ["", "", "", ""], "9304": ["", "", "", ""], "9305": ["", "", "", ""], "9306": ["", "", "", ""], "9307": ["", "", "", ""], "9308": ["", "", "", ""], "9309": ["", "", "", ""], "9310": ["", "", "", ""], "9311": ["", "", "", ""], "9312": ["", "", "", ""], "9313": ["", "", "", ""], "9314": ["", "", "", ""], "9315": ["", "", "", ""], "9316": ["", "", "", ""], "9317": ["", "", "", ""], "9318": ["", "", "", ""], "9319": ["", "", "", ""], "9320": ["", "", "", ""], "9321": ["", "", "", ""], "9322": ["", "", "", ""], "9323": ["", "", "", ""], "9324": ["", "", "", ""], "9325": ["", "", "", ""], "9326": ["", "", "", ""], "9327": ["", "", "", ""], "9328": ["", "", "", ""], "9329": ["", "", "", ""], "9330": ["", "", "", ""], "9331": ["", "", "", ""], "9332": ["", "", "", ""], "9333": ["", "", "", ""], "9334": ["", "", "", ""], "9335": ["", "", "", ""], "9336": ["", "", "", ""], "9337": ["", "", "", ""], "9338": ["", "", "", ""], "9339": ["", "", "", ""], "9340": ["", "", "", ""], "9341": ["", "", "", ""], "9342": ["", "", "", ""], "9343": ["", "", "", ""], "9344": ["", "", "", ""], "9345": ["", "", "", ""], "9346": ["", "", "", ""], "9347": ["", "", "", ""], "9348": ["", "", "", ""], "9349": ["", "", "", ""], "9350": ["", "", "", ""], "9351": ["", "", "", ""], "9352": ["", "", "", ""], "9353": ["", "", "", ""], "9354": ["", "", "", ""], "9355": ["", "", "", ""], "9356": ["", "", "", ""], "9357": ["", "", "", ""], "9358": ["", "", "", ""], "9359": ["", "", "", ""], "9360": ["", "", "", ""], "9361": ["", "", "", ""], "9362": ["", "", "", ""], "9363": ["", "", "", ""], "9364": ["", "", "", ""], "9365": ["", "", "", ""], "9366": ["", "", "", ""], "9367": ["", "", "", ""], "9368": ["", "", "", ""], "9369": ["", "", "", ""], "9370": ["", "", "", ""], "9371": ["", "", "", ""], "9372": ["", "", "", ""], "9373": ["", "", "", ""], "9374": ["", "", "", ""], "9375": ["", "", "", ""], "9376": ["", "", "", ""], "9377": ["", "", "", ""], "9378": ["", "", "", ""], "9379": ["", "", "", ""], "9380": ["", "", "", ""], "9381": ["", "", "", ""], "9382": ["", "", "", ""], "9383": ["", "", "", ""], "9384": ["", "", "", ""], "9385": ["", "", "", ""], "9386": ["", "", "", ""], "9387": ["", "", "", ""], "9388": ["", "", "", ""], "9389": ["", "", "", ""], "9390": ["", "", "", ""], "9391": ["", "", "", ""], "9392": ["", "", "", ""], "9393": ["", "", "", ""], "9394": ["", "", "", ""], "9395": ["", "", "", ""], "9396": ["", "", "", ""], "9397": ["", "", "", ""], "9398": ["", "", "", ""], "9399": ["", "", "", ""], "9400": ["", "", "", ""], "9401": ["", "", "", ""], "9402": ["", "", "", ""], "9403": ["", "", "", ""], "9404": ["", "", "", ""], "9405": ["", "", "", ""], "9406": ["", "", "", ""], "9407": ["", "", "", ""], "9408": ["", "", "", ""], "9409": ["", "", "", ""], "9410": ["", "", "", ""], "9411": ["", "", "", ""], "9412": ["", "", "", ""], "9413": ["", "", "", ""], "9414": ["", "", "", ""], "9415": ["", "", "", ""], "9416": ["", "", "", ""], "9417": ["", "", "", ""], "9418": ["", "", "", ""], "9419": ["", "", "", ""], "9420": ["", "", "", ""], "9421": ["", "", "", ""], "9422": ["", "", "", ""], "9423": ["", "", "", ""], "9424": ["", "", "", ""], "9425": ["", "", "", ""], "9426": ["", "", "", ""], "9427": ["", "", "", ""], "9428": ["", "", "", ""], "9429": ["", "", "", ""], "9430": ["", "", "", ""], "9431": ["", "", "", ""], "9432": ["", "", "", ""], "9433": ["", "", "", ""], "9434": ["", "", "", ""], "9435": ["", "", "", ""], "9436": ["", "", "", ""], "9437": ["", "", "", ""], "9438": ["", "", "", ""], "9439": ["", "", "", ""], "9440": ["", "", "", ""], "9441": ["", "", "", ""], "9442": ["", "", "", ""], "9443": ["", "", "", ""], "9444": ["", "", "", ""], "9445": ["", "", "", ""], "9446": ["", "", "", ""], "9447": ["", "", "", ""], "9448": ["", "", "", ""], "9449": ["", "", "", ""], "9450": ["", "", "", ""], "9451": ["", "", "", ""], "9452": ["", "", "", ""], "9453": ["", "", "", ""], "9454": ["", "", "", ""], "9455": ["", "", "", ""], "9456": ["", "", "", ""], "9457": ["", "", "", ""], "9458": ["", "", "", ""], "9459": ["", "", "", ""], "9460": ["", "", "", ""], "9461": ["", "", "", ""], "9462": ["", "", "", ""], "9463": ["", "", "", ""], "9464": ["", "", "", ""], "9465": ["", "", "", ""], "9466": ["", "", "", ""], "9467": ["", "", "", ""], "9468": ["", "", "", ""], "9469": ["", "", "", ""], "9470": ["", "", "", ""], "9471": ["", "", "", ""], "9472": ["", "", "", ""], "9473": ["", "", "", ""], "9474": ["", "", "", ""], "9475": ["", "", "", ""], "9476": ["", "", "", ""], "9477": ["", "", "", ""], "9478": ["", "", "", ""], "9479": ["", "", "", ""], "9480": ["", "", "", ""], "9481": ["", "", "", ""], "9482": ["", "", "", ""], "9483": ["", "", "", ""], "9484": ["", "", "", ""], "9485": ["", "", "", ""], "9486": ["", "", "", ""], "9487": ["", "", "", ""], "9488": ["", "", "", ""], "9489": ["", "", "", ""], "9490": ["", "", "", ""], "9491": ["", "", "", ""], "9492": ["", "", "", ""], "9493": ["", "", "", ""], "9494": ["", "", "", ""], "9495": ["", "", "", ""], "9496": ["", "", "", ""], "9497": ["", "", "", ""], "9498": ["", "", "", ""], "9499": ["", "", "", ""], "9500": ["", "", "", ""], "9501": ["", "", "", ""], "9502": ["", "", "", ""], "9503": ["", "", "", ""], "9504": ["", "", "", ""], "9505": ["", "", "", ""], "9506": ["", "", "", ""], "9507": ["", "", "", ""], "9508": ["", "", "", ""], "9509": ["", "", "", ""], "9510": ["", "", "", ""], "9511": ["", "", "", ""], "9512": ["", "", "", ""], "9513": ["", "", "", ""], "9514": ["", "", "", ""], "9515": ["", "", "", ""], "9516": ["", "", "", ""], "9517": ["", "", "", ""], "9518": ["", "", "", ""], "9519": ["", "", "", ""], "9520": ["", "", "", ""], "9521": ["", "", "", ""], "9522": ["", "", "", ""], "9523": ["", "", "", ""], "9524": ["", "", "", ""], "9525": ["", "", "", ""], "9526": ["", "", "", ""], "9527": ["", "", "", ""], "9528": ["", "", "", ""], "9529": ["", "", "", ""], "9530": ["", "", "", ""], "9531": ["", "", "", ""], "9532": ["", "", "", ""], "9533": ["", "", "", ""], "9534": ["", "", "", ""], "9535": ["", "", "", ""], "9536": ["", "", "", ""], "9537": ["", "", "", ""], "9538": ["", "", "", ""], "9539": ["", "", "", ""], "9540": ["", "", "", ""], "9541": ["", "", "", ""], "9542": ["", "", "", ""], "9543": ["", "", "", ""], "9544": ["", "", "", ""], "9545": ["", "", "", ""], "9546": ["", "", "", ""], "9547": ["", "", "", ""], "9548": ["", "", "", ""], "9549": ["", "", "", ""], "9550": ["", "", "", ""], "9551": ["", "", "", ""], "9552": ["", "", "", ""], "9553": ["", "", "", ""], "9554": ["", "", "", ""], "9555": ["", "", "", ""], "9556": ["", "", "", ""], "9557": ["", "", "", ""], "9558": ["", "", "", ""], "9559": ["", "", "", ""], "9560": ["", "", "", ""], "9561": ["", "", "", ""], "9562": ["", "", "", ""], "9563": ["", "", "", ""], "9564": ["", "", "", ""], "9565": ["", "", "", ""], "9566": ["", "", "", ""], "9567": ["", "", "", ""], "9568": ["", "", "", ""], "9569": ["", "", "", ""], "9570": ["", "", "", ""], "9571": ["", "", "", ""], "9572": ["", "", "", ""], "9573": ["", "", "", ""], "9574": ["", "", "", ""], "9575": ["", "", "", ""], "9576": ["", "", "", ""], "9577": ["", "", "", ""], "9578": ["", "", "", ""], "9579": ["", "", "", ""], "9580": ["", "", "", ""], "9581": ["", "", "", ""], "9582": ["", "", "", ""], "9583": ["", "", "", ""], "9584": ["", "", "", ""], "9585": ["", "", "", ""], "9586": ["", "", "", ""], "9587": ["", "", "", ""], "9588": ["", "", "", ""], "9589": ["", "", "", ""], "9590": ["", "", "", ""], "9591": ["", "", "", ""], "9592": ["", "", "", ""], "9593": ["", "", "", ""], "9594": ["", "", "", ""], "9595": ["", "", "", ""], "9596": ["", "", "", ""], "9597": ["", "", "", ""], "9598": ["", "", "", ""], "9599": ["", "", "", ""], "9600": ["", "", "", ""], "9601": ["", "", "", ""], "9602": ["", "", "", ""], "9603": ["", "", "", ""], "9604": ["", "", "", ""], "9605": ["", "", "", ""], "9606": ["", "", "", ""], "9607": ["", "", "", ""], "9608": ["", "", "", ""], "9609": ["", "", "", ""], "9610": ["", "", "", ""], "9611": ["", "", "", ""], "9612": ["", "", "", ""], "9613": ["", "", "", ""], "9614": ["", "", "", ""], "9615": ["", "", "", ""], "9616": ["", "", "", ""], "9617": ["", "", "", ""], "9618": ["", "", "", ""], "9619": ["", "", "", ""], "9620": ["", "", "", ""], "9621": ["", "", "", ""], "9622": ["", "", "", ""], "9623": ["", "", "", ""], "9624": ["", "", "", ""], "9625": ["", "", "", ""], "9626": ["", "", "", ""], "9627": ["", "", "", ""], "9628": ["", "", "", ""], "9629": ["", "", "", ""], "9630": ["", "", "", ""], "9631": ["", "", "", ""], "9632": ["", "", "", ""], "9633": ["", "", "", ""], "9634": ["", "", "", ""], "9635": ["", "", "", ""], "9636": ["", "", "", ""], "9637": ["", "", "", ""], "9638": ["", "", "", ""], "9639": ["", "", "", ""], "9640": ["", "", "", ""], "9641": ["", "", "", ""], "9642": ["", "", "", ""], "9643": ["", "", "", ""], "9644": ["", "", "", ""], "9645": ["", "", "", ""], "9646": ["", "", "", ""], "9647": ["", "", "", ""], "9648": ["", "", "", ""], "9649": ["", "", "", ""], "9650": ["", "", "", ""], "9651": ["", "", "", ""], "9652": ["", "", "", ""], "9653": ["", "", "", ""], "9654": ["", "", "", ""], "9655": ["", "", "", ""], "9656": ["", "", "", ""], "9657": ["", "", "", ""], "9658": ["", "", "", ""], "9659": ["", "", "", ""], "9660": ["", "", "", ""], "9661": ["", "", "", ""], "9662": ["", "", "", ""], "9663": ["", "", "", ""], "9664": ["", "", "", ""], "9665": ["", "", "", ""], "9666": ["", "", "", ""], "9667": ["", "", "", ""], "9668": ["", "", "", ""], "9669": ["", "", "", ""], "9670": ["", "", "", ""], "9671": ["", "", "", ""], "9672": ["", "", "", ""], "9673": ["", "", "", ""], "9674": ["", "", "", ""], "9675": ["", "", "", ""], "9676": ["", "", "", ""], "9677": ["", "", "", ""], "9678": ["", "", "", ""], "9679": ["", "", "", ""], "9680": ["", "", "", ""], "9681": ["", "", "", ""], "9682": ["", "", "", ""], "9683": ["", "", "", ""], "9684": ["", "", "", ""], "9685": ["", "", "", ""], "9686": ["", "", "", ""], "9687": ["", "", "", ""], "9688": ["", "", "", ""], "9689": ["", "", "", ""], "9690": ["", "", "", ""], "9691": ["", "", "", ""], "9692": ["", "", "", ""], "9693": ["", "", "", ""], "9694": ["", "", "", ""], "9695": ["", "", "", ""], "9696": ["", "", "", ""], "9697": ["", "", "", ""], "9698": ["", "", "", ""], "9699": ["", "", "", ""], "9700": ["", "", "", ""], "9701": ["", "", "", ""], "9702": ["", "", "", ""], "9703": ["", "", "", ""], "9704": ["", "", "", ""], "9705": ["", "", "", ""], "9706": ["", "", "", ""], "9707": ["", "", "", ""], "9708": ["", "", "", ""], "9709": ["", "", "", ""], "9710": ["", "", "", ""], "9711": ["", "", "", ""], "9712": ["", "", "", ""], "9713": ["", "", "", ""], "9714": ["", "", "", ""], "9715": ["", "", "", ""], "9716": ["", "", "", ""], "9717": ["", "", "", ""], "9718": ["", "", "", ""], "9719": ["", "", "", ""], "9720": ["", "", "", ""], "9721": ["", "", "", ""], "9722": ["", "", "", ""], "9723": ["", "", "", ""], "9724": ["", "", "", ""], "9725": ["", "", "", ""], "9726": ["", "", "", ""], "9727": ["", "", "", ""], "9728": ["", "", "", ""], "9729": ["", "", "", ""], "9730": ["", "", "", ""], "9731": ["", "", "", ""], "9732": ["", "", "", ""], "9733": ["", "", "", ""], "9734": ["", "", "", ""], "9735": ["", "", "", ""], "9736": ["", "", "", ""], "9737": ["", "", "", ""], "9738": ["", "", "", ""], "9739": ["", "", "", ""], "9740": ["", "", "", ""], "9741": ["", "", "", ""], "9742": ["", "", "", ""], "9743": ["", "", "", ""], "9744": ["", "", "", ""], "9745": ["", "", "", ""], "9746": ["", "", "", ""], "9747": ["", "", "", ""], "9748": ["", "", "", ""], "9749": ["", "", "", ""], "9750": ["", "", "", ""], "9751": ["", "", "", ""], "9752": ["", "", "", ""], "9753": ["", "", "", ""], "9754": ["", "", "", ""], "9755": ["", "", "", ""], "9756": ["", "", "", ""], "9757": ["", "", "", ""], "9758": ["", "", "", ""], "9759": ["", "", "", ""], "9760": ["", "", "", ""], "9761": ["", "", "", ""], "9762": ["", "", "", ""], "9763": ["", "", "", ""], "9764": ["", "", "", ""], "9765": ["", "", "", ""], "9766": ["", "", "", ""], "9767": ["", "", "", ""], "9768": ["", "", "", ""], "9769": ["", "", "", ""], "9770": ["", "", "", ""], "9771": ["", "", "", ""], "9772": ["", "", "", ""], "9773": ["", "", "", ""], "9774": ["", "", "", ""], "9775": ["", "", "", ""], "9776": ["", "", "", ""], "9777": ["", "", "", ""], "9778": ["", "", "", ""], "9779": ["", "", "", ""], "9780": ["", "", "", ""], "9781": ["", "", "", ""], "9782": ["", "", "", ""], "9783": ["", "", "", ""], "9784": ["", "", "", ""], "9785": ["", "", "", ""], "9786": ["", "", "", ""], "9787": ["", "", "", ""], "9788": ["", "", "", ""], "9789": ["", "", "", ""], "9790": ["", "", "", ""], "9791": ["", "", "", ""], "9792": ["", "", "", ""], "9793": ["", "", "", ""], "9794": ["", "", "", ""], "9795": ["", "", "", ""], "9796": ["", "", "", ""], "9797": ["", "", "", ""], "9798": ["", "", "", ""], "9799": ["", "", "", ""], "9800": ["", "", "", ""], "9801": ["", "", "", ""], "9802": ["", "", "", ""], "9803": ["", "", "", ""], "9804": ["", "", "", ""], "9805": ["", "", "", ""], "9806": ["", "", "", ""], "9807": ["", "", "", ""], "9808": ["", "", "", ""], "9809": ["", "", "", ""], "9810": ["", "", "", ""], "9811": ["", "", "", ""], "9812": ["", "", "", ""], "9813": ["", "", "", ""], "9814": ["", "", "", ""], "9815": ["", "", "", ""], "9816": ["", "", "", ""], "9817": ["", "", "", ""], "9818": ["", "", "", ""], "9819": ["", "", "", ""], "9820": ["", "", "", ""], "9821": ["", "", "", ""], "9822": ["", "", "", ""], "9823": ["", "", "", ""], "9824": ["", "", "", ""], "9825": ["", "", "", ""], "9826": ["", "", "", ""], "9827": ["", "", "", ""], "9828": ["", "", "", ""], "9829": ["", "", "", ""], "9830": ["", "", "", ""], "9831": ["", "", "", ""], "9832": ["", "", "", ""], "9833": ["", "", "", ""], "9834": ["", "", "", ""], "9835": ["", "", "", ""], "9836": ["", "", "", ""], "9837": ["", "", "", ""], "9838": ["", "", "", ""], "9839": ["", "", "", ""], "9840": ["", "", "", ""], "9841": ["", "", "", ""], "9842": ["", "", "", ""], "9843": ["", "", "", ""], "9844": ["", "", "", ""], "9845": ["", "", "", ""], "9846": ["", "", "", ""], "9847": ["", "", "", ""], "9848": ["", "", "", ""], "9849": ["", "", "", ""], "9850": ["", "", "", ""], "9851": ["", "", "", ""], "9852": ["", "", "", ""], "9853": ["", "", "", ""], "9854": ["", "", "", ""], "9855": ["", "", "", ""], "9856": ["", "", "", ""], "9857": ["", "", "", ""], "9858": ["", "", "", ""], "9859": ["", "", "", ""], "9860": ["", "", "", ""], "9861": ["", "", "", ""], "9862": ["", "", "", ""], "9863": ["", "", "", ""], "9864": ["", "", "", ""], "9865": ["", "", "", ""], "9866": ["", "", "", ""], "9867": ["", "", "", ""], "9868": ["", "", "", ""], "9869": ["", "", "", ""], "9870": ["", "", "", ""], "9871": ["", "", "", ""], "9872": ["", "", "", ""], "9873": ["", "", "", ""], "9874": ["", "", "", ""], "9875": ["", "", "", ""], "9876": ["", "", "", ""], "9877": ["", "", "", ""], "9878": ["", "", "", ""], "9879": ["", "", "", ""], "9880": ["", "", "", ""], "9881": ["", "", "", ""], "9882": ["", "", "", ""], "9883": ["", "", "", ""], "9884": ["", "", "", ""], "9885": ["", "", "", ""], "9886": ["", "", "", ""], "9887": ["", "", "", ""], "9888": ["", "", "", ""], "9889": ["", "", "", ""], "9890": ["", "", "", ""], "9891": ["", "", "", ""], "9892": ["", "", "", ""], "9893": ["", "", "", ""], "9894": ["", "", "", ""], "9895": ["", "", "", ""], "9896": ["", "", "", ""], "9897": ["", "", "", ""], "9898": ["", "", "", ""], "9899": ["", "", "", ""], "9900": ["", "", "", ""], "9901": ["", "", "", ""], "9902": ["", "", "", ""], "9903": ["", "", "", ""], "9904": ["", "", "", ""], "9905": ["", "", "", ""], "9906": ["", "", "", ""], "9907": ["", "", "", ""], "9908": ["", "", "", ""], "9909": ["", "", "", ""], "9910": ["", "", "", ""], "9911": ["", "", "", ""], "9912": ["", "", "", ""], "9913": ["", "", "", ""], "9914": ["", "", "", ""], "9915": ["", "", "", ""], "9916": ["", "", "", ""], "9917": ["", "", "", ""], "9918": ["", "", "", ""], "9919": ["", "", "", ""], "9920": ["", "", "", ""], "9921": ["", "", "", ""]} \ No newline at end of file diff --git a/Ins/log.txt b/Ins/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c0a7b038595e01f99a8c650486887411588b222 --- /dev/null +++ b/Ins/log.txt @@ -0,0 +1,2821 @@ +W0404 07:59:56.954000 140117560674112 torch/distributed/run.py:779] +W0404 07:59:56.954000 140117560674112 torch/distributed/run.py:779] ***************************************** +W0404 07:59:56.954000 140117560674112 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0404 07:59:56.954000 140117560674112 torch/distributed/run.py:779] ***************************************** +{'seed': 42, 'base_model': '/home/jovyan/workspace/Llama-7b', 'output_dir': './Ins', 'optim': 'adamw_torch', 'epochs': 1, 'learning_rate': 0.0005, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 2, 'logging_step': 10, 'model_max_length': 1024, 'weight_decay': 0.01, 'lora_r': 8, 'lora_alpha': 32, 'lora_dropout': 0.05, 'lora_target_modules': 'q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj', 'lora_modules_to_save': 'embed_tokens,lm_head', 'resume_from_checkpoint': None, 'warmup_ratio': 0.01, 'lr_scheduler_type': 'cosine', 'save_and_eval_strategy': 'epoch', 'save_and_eval_steps': 1000, 'fp16': False, 'bf16': True, 'deepspeed': './config/ds_z2_bf16.json', 'remove_unused_columns': False, 'data_path': '/home/jovyan/workspace', 'tasks': 'seqrec,itemsearch,inters2title,inters2description,preferenceobtain,item2index,index2item,intertitles2item,query2item', 'train_data_sample_num': '0,0,0,0,0,0,0,0,0', 'dataset': 'Instruments', 'index_file': '.index.json', 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': 2, 'max_his_len': 20, 'add_prefix': False, 'his_sep': ', ', 'only_train_response': True, 'train_prompt_sample_num': '1,1,1,1,1,1,1,1,1', 'valid_prompt_id': 0, 'sample_valid': True, 'valid_prompt_sample_num': 2, 'lr': 0.001, 'batch_size': 1024, 'num_workers': 4, 'eval_step': 50, 'learner': 'AdamW', 'dropout_prob': 0.0, 'bn': False, 'loss_type': 'mse', 'kmeans_init': False, 'kmeans_iters': 100, 'sk_epsilons': [0.0, 0.0, 0.0, 0.0], 'sk_iters': 50, 'device': 'cuda:1', 'num_emb_list': [256, 256, 256, 256], 'e_dim': 32, 'quant_loss_weight': 1.0, 'layers': [2048, 1024, 512, 256, 128, 64], 'ckpt_dir': '', 'warmup': 5, 'rqvae_model': '/home/jovyan/workspace/LC-Rec/index/Ins/Apr-04-2025_07-12-04/best_collision_model.pth'} +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message + Loading checkpoint shards: 0%| | 0/2 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message + Loading checkpoint shards: 0%| | 0/2 [00:00", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/Ins/tokenizer_config.json b/Ins/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..af2930551fdc0a50e9fb9b05d370d7743e0ade97 --- /dev/null +++ b/Ins/tokenizer_config.json @@ -0,0 +1,8236 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": true, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": true + }, + "32000": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32001": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32002": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32003": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32004": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32005": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32006": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32007": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32008": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32009": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32010": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32011": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32012": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32013": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32014": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32015": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32016": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32017": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32018": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32019": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32020": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32021": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32022": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32023": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32024": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32025": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32026": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32027": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32028": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32029": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32030": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32031": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32032": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32033": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32034": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32035": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32036": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32037": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32038": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32039": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32040": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32041": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32042": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32043": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32044": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32045": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32046": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32047": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32048": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32049": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32050": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32051": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32052": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32053": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32054": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32055": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32056": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32057": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32058": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32059": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32060": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32061": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32062": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32063": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32064": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32065": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32066": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32067": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32068": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32069": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32070": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32071": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32072": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32073": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32074": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32075": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32076": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32077": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32078": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32079": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32080": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32081": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32082": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32083": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32084": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32085": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32086": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32087": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32088": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32089": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32090": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32091": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32092": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32093": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32094": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32095": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32096": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32097": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32098": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32099": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32100": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32101": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32102": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32103": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32104": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32105": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32106": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32107": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32108": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32109": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32110": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32111": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32112": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32113": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32114": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32115": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32116": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32117": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32118": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32119": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32120": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32121": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32122": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32123": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32124": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32125": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32126": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32127": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32128": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32129": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32130": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32131": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32132": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32133": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32134": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32135": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32136": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32137": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32138": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32139": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32140": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32141": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32142": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32143": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32144": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32145": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32146": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32147": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32148": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32149": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32150": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32151": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32152": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32153": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32154": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32155": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32156": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32157": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32158": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32159": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32160": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32161": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32162": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32163": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32164": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32165": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32166": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32167": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32168": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32169": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32170": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32171": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32172": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32173": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32174": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32175": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32176": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32177": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32178": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32179": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32180": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32181": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32182": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32183": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32184": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32185": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32186": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32187": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32188": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32189": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32190": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32191": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32192": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32193": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32194": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32195": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32196": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32197": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32198": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32199": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32200": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32201": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32202": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32203": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32204": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32205": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32206": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32207": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32208": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32209": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32210": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32211": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32212": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32213": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32214": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32215": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32216": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32217": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32218": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32219": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32220": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32221": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32222": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32223": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32224": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32225": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32226": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32227": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32228": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32229": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32230": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32231": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32232": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32233": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32234": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32235": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32236": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32237": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32238": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32239": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32240": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32241": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32242": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32243": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32244": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32245": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32246": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32247": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32248": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32249": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32250": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32251": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32252": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32253": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32254": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32255": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32256": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32257": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32258": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32259": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32260": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32261": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32262": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32263": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32264": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32265": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32266": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32267": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32268": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32269": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32270": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32271": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32272": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32273": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32274": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32275": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32276": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32277": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32278": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32279": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32280": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32281": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32282": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32283": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32284": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32285": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32286": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32287": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32288": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32289": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32290": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32291": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32292": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32293": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32294": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32295": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32296": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32297": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32298": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32299": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32300": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32301": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32302": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32303": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32304": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32305": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32306": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32307": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32308": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32309": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32310": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32311": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32312": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32313": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32314": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32315": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32316": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32317": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32318": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32319": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32320": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32321": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32322": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32323": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32324": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32325": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32326": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32327": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32328": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32329": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32330": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32331": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32332": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32333": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32334": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32335": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32336": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32337": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32338": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32339": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32340": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32341": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32342": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32343": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32344": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32345": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32346": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32347": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32348": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32349": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32350": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32351": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32352": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32353": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32354": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32355": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32356": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32357": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32358": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32359": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32360": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32361": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32362": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32363": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32364": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32365": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32366": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32367": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32368": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32369": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32370": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32371": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32372": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32373": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32374": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32375": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32376": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32377": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32378": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32379": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32380": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32381": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32382": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32383": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32384": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32385": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32386": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32387": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32388": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32389": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32390": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32391": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32392": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32393": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32394": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32395": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32396": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32397": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32398": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32399": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32400": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32401": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32402": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32403": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32404": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32405": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32406": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32407": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32408": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32409": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32410": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32411": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32412": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32413": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32414": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32415": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32416": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32417": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32418": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32419": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32420": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32421": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32422": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32423": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32424": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32425": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32426": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32427": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32428": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32429": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32430": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32431": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32432": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32433": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32434": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32435": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32436": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32437": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32438": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32439": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32440": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32441": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32442": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32443": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32444": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32445": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32446": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32447": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32448": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32449": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32450": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32451": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32452": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32453": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32454": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32455": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32456": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32457": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32458": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32459": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32460": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32461": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32462": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32463": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32464": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32465": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32466": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32467": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32468": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32469": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32470": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32471": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32472": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32473": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32474": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32475": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32476": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32477": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32478": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32479": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32480": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32481": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32482": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32483": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32484": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32485": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32486": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32487": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32488": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32489": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32490": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32491": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32492": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32493": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32494": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32495": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32496": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32497": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32498": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32499": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32500": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32501": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32502": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32503": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32504": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32505": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32506": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32507": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32508": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32509": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32510": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32511": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32512": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32513": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32514": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32515": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32516": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32517": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32518": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32519": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32520": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32521": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32522": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32523": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32524": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32525": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32526": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32527": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32528": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32529": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32530": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32531": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32532": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32533": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32534": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32535": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32536": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32537": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32538": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32539": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32540": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32541": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32542": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32543": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32544": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32545": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32546": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32547": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32548": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32549": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32550": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32551": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32552": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32553": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32554": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32555": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32556": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32557": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32558": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32559": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32560": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32561": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32562": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32563": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32564": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32565": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32566": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32567": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32568": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32569": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32570": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32571": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32572": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32573": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32574": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32575": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32576": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32577": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32578": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32579": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32580": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32581": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32582": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32583": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32584": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32585": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32586": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32587": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32588": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32589": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32590": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32591": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32592": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32593": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32594": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32595": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32596": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32597": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32598": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32599": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32600": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32601": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32602": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32603": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32604": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32605": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32606": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32607": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32608": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32609": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32610": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32611": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32612": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32613": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32614": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32615": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32616": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32617": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32618": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32619": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32620": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32621": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32622": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32623": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32624": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32625": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32626": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32627": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32628": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32629": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32630": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32631": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32632": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32633": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32634": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32635": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32636": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32637": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32638": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32639": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32640": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32641": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32642": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32643": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32644": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32645": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32646": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32647": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32648": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32649": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32650": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32651": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32652": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32653": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32654": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32655": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32656": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32657": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32658": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32659": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32660": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32661": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32662": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32663": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32664": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32665": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32666": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32667": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32668": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32669": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32670": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32671": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32672": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32673": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32674": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32675": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32676": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32677": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32678": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32679": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32680": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32681": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32682": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32683": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32684": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32685": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32686": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32687": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32688": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32689": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32690": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32691": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32692": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32693": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32694": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32695": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32696": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32697": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32698": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32699": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32700": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32701": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32702": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32703": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32704": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32705": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32706": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32707": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32708": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32709": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32710": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32711": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32712": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32713": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32714": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32715": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32716": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32717": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32718": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32719": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32720": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32721": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32722": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32723": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32724": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32725": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32726": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32727": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32728": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32729": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32730": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32731": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32732": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32733": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32734": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32735": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32736": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32737": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32738": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32739": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32740": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32741": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32742": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32743": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32744": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32745": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32746": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32747": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32748": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32749": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32750": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32751": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32752": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32753": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32754": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32755": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32756": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32757": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32758": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32759": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32760": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32761": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32762": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32763": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32764": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32765": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32766": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32767": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32768": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32769": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32770": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32771": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32772": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32773": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32774": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32775": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32776": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32777": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32778": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32779": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32780": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32781": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32782": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32783": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32784": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32785": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32786": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32787": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32788": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32789": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32790": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32791": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32792": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32793": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32794": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32795": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32796": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32797": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32798": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32799": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32800": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32801": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32802": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32803": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32804": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32805": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32806": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32807": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32808": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32809": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32810": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32811": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32812": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32813": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32814": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32815": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32816": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32817": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32818": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32819": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32820": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32821": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32822": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32823": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32824": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32825": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32826": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32827": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32828": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32829": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32830": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32831": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32832": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32833": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32834": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32835": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32836": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32837": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32838": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32839": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32840": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32841": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32842": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32843": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32844": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32845": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32846": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32847": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32848": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32849": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32850": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32851": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32852": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32853": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32854": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32855": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32856": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32857": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32858": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32859": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32860": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32861": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32862": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32863": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32864": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32865": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32866": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32867": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32868": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32869": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32870": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32871": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32872": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32873": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32874": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32875": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32876": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32877": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32878": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32879": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32880": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32881": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32882": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32883": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32884": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32885": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32886": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32887": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32888": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32889": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32890": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32891": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32892": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32893": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32894": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32895": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32896": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32897": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32898": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32899": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32900": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32901": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32902": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32903": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32904": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32905": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32906": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32907": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32908": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32909": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32910": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32911": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32912": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32913": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32914": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32915": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32916": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32917": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32918": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32919": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32920": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32921": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32922": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32923": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32924": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32925": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32926": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32927": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32928": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32929": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32930": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32931": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32932": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32933": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32934": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32935": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32936": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32937": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32938": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32939": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32940": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32941": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32942": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32943": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32944": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32945": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32946": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32947": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32948": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32949": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32950": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32951": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32952": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32953": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32954": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32955": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32956": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32957": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32958": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32959": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32960": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32961": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32962": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32963": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32964": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32965": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32966": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32967": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32968": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32969": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32970": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32971": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32972": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32973": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32974": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32975": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32976": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32977": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32978": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32979": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32980": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32981": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32982": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32983": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32984": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32985": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32986": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32987": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32988": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32989": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32990": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32991": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32992": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32993": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32994": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32995": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32996": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32997": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32998": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "32999": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33000": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33001": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33002": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33003": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33004": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33005": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33006": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33007": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33008": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33009": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33010": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33011": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33012": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33013": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33014": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33015": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33016": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33017": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33018": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33019": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33020": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33021": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33022": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + }, + "33023": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\\n\\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don\\'t know the answer to a question, please don\\'t share false information.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\n' + content.strip() + '\\n<>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/Ins/trainer_state.json b/Ins/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..14832936abd45fcd122019d6df5041e14030d2ed --- /dev/null +++ b/Ins/trainer_state.json @@ -0,0 +1,6819 @@ +{ + "best_metric": 2.729660987854004, + "best_model_checkpoint": "./Ins/checkpoint-9678", + "epoch": 1.0, + "eval_steps": 1000, + "global_step": 9678, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0010332713370531101, + "grad_norm": 9.612689971923828, + "learning_rate": 5.1546391752577315e-05, + "loss": 4.2162, + "step": 10 + }, + { + "epoch": 0.0020665426741062203, + "grad_norm": 6.56850004196167, + "learning_rate": 0.00010309278350515463, + "loss": 3.388, + "step": 20 + }, + { + "epoch": 0.0030998140111593306, + "grad_norm": 4.049291610717773, + "learning_rate": 0.00015463917525773197, + "loss": 2.7081, + "step": 30 + }, + { + "epoch": 0.0041330853482124405, + "grad_norm": 2.604156970977783, + "learning_rate": 0.00020618556701030926, + "loss": 2.8688, + "step": 40 + }, + { + "epoch": 0.0051663566852655505, + "grad_norm": 1.8748372793197632, + "learning_rate": 0.00025773195876288655, + "loss": 2.6251, + "step": 50 + }, + { + "epoch": 0.006199628022318661, + "grad_norm": 2.638007164001465, + "learning_rate": 0.00030927835051546395, + "loss": 2.5802, + "step": 60 + }, + { + "epoch": 0.007232899359371771, + "grad_norm": 1.8762423992156982, + "learning_rate": 0.00036082474226804123, + "loss": 2.4978, + "step": 70 + }, + { + "epoch": 0.008266170696424881, + "grad_norm": 1.7499455213546753, + "learning_rate": 0.0004123711340206185, + "loss": 2.4351, + "step": 80 + }, + { + "epoch": 0.009299442033477991, + "grad_norm": 1.4334783554077148, + "learning_rate": 0.0004639175257731959, + "loss": 2.4811, + "step": 90 + }, + { + "epoch": 0.010332713370531101, + "grad_norm": 1.4335546493530273, + "learning_rate": 0.0004999998790431266, + "loss": 2.3653, + "step": 100 + }, + { + "epoch": 0.01136598470758421, + "grad_norm": 1.379575490951538, + "learning_rate": 0.0004999977287019656, + "loss": 2.2769, + "step": 110 + }, + { + "epoch": 0.012399256044637322, + "grad_norm": 1.357542872428894, + "learning_rate": 0.0004999928904568952, + "loss": 2.1347, + "step": 120 + }, + { + "epoch": 0.013432527381690432, + "grad_norm": 1.6458231210708618, + "learning_rate": 0.0004999853643599349, + "loss": 2.1943, + "step": 130 + }, + { + "epoch": 0.014465798718743542, + "grad_norm": 1.7004691362380981, + "learning_rate": 0.0004999751504920031, + "loss": 2.204, + "step": 140 + }, + { + "epoch": 0.015499070055796652, + "grad_norm": 1.3042373657226562, + "learning_rate": 0.0004999622489629164, + "loss": 2.0817, + "step": 150 + }, + { + "epoch": 0.016532341392849762, + "grad_norm": 1.5508415699005127, + "learning_rate": 0.0004999466599113884, + "loss": 2.115, + "step": 160 + }, + { + "epoch": 0.017565612729902872, + "grad_norm": 1.3333240747451782, + "learning_rate": 0.0004999283835050278, + "loss": 2.1762, + "step": 170 + }, + { + "epoch": 0.018598884066955982, + "grad_norm": 1.3132269382476807, + "learning_rate": 0.0004999074199403377, + "loss": 2.0493, + "step": 180 + }, + { + "epoch": 0.019632155404009092, + "grad_norm": 1.250373363494873, + "learning_rate": 0.000499883769442712, + "loss": 2.2234, + "step": 190 + }, + { + "epoch": 0.020665426741062202, + "grad_norm": 1.2733889818191528, + "learning_rate": 0.0004998574322664343, + "loss": 1.9751, + "step": 200 + }, + { + "epoch": 0.02169869807811531, + "grad_norm": 0.9156686663627625, + "learning_rate": 0.0004998284086946743, + "loss": 2.1314, + "step": 210 + }, + { + "epoch": 0.02273196941516842, + "grad_norm": 1.0921283960342407, + "learning_rate": 0.0004997966990394851, + "loss": 1.8402, + "step": 220 + }, + { + "epoch": 0.023765240752221535, + "grad_norm": 1.2582165002822876, + "learning_rate": 0.0004997623036417998, + "loss": 2.1236, + "step": 230 + }, + { + "epoch": 0.024798512089274645, + "grad_norm": 1.1085331439971924, + "learning_rate": 0.0004997252228714278, + "loss": 2.0778, + "step": 240 + }, + { + "epoch": 0.025831783426327755, + "grad_norm": 1.096815824508667, + "learning_rate": 0.0004996854571270512, + "loss": 2.146, + "step": 250 + }, + { + "epoch": 0.026865054763380865, + "grad_norm": 1.0710986852645874, + "learning_rate": 0.0004996430068362196, + "loss": 1.9776, + "step": 260 + }, + { + "epoch": 0.027898326100433975, + "grad_norm": 1.4760586023330688, + "learning_rate": 0.0004995978724553464, + "loss": 1.9536, + "step": 270 + }, + { + "epoch": 0.028931597437487085, + "grad_norm": 1.3000599145889282, + "learning_rate": 0.0004995500544697036, + "loss": 1.7954, + "step": 280 + }, + { + "epoch": 0.029964868774540195, + "grad_norm": 1.2158455848693848, + "learning_rate": 0.0004994995533934163, + "loss": 1.8429, + "step": 290 + }, + { + "epoch": 0.030998140111593304, + "grad_norm": 1.1453113555908203, + "learning_rate": 0.0004994463697694579, + "loss": 2.0769, + "step": 300 + }, + { + "epoch": 0.032031411448646414, + "grad_norm": 0.7675238847732544, + "learning_rate": 0.0004993905041696433, + "loss": 1.903, + "step": 310 + }, + { + "epoch": 0.033064682785699524, + "grad_norm": 1.1099224090576172, + "learning_rate": 0.0004993319571946235, + "loss": 1.9536, + "step": 320 + }, + { + "epoch": 0.034097954122752634, + "grad_norm": 1.617551326751709, + "learning_rate": 0.0004992707294738786, + "loss": 1.9095, + "step": 330 + }, + { + "epoch": 0.035131225459805744, + "grad_norm": 1.486296534538269, + "learning_rate": 0.0004992068216657115, + "loss": 1.8174, + "step": 340 + }, + { + "epoch": 0.036164496796858854, + "grad_norm": 1.4763352870941162, + "learning_rate": 0.0004991402344572409, + "loss": 1.7963, + "step": 350 + }, + { + "epoch": 0.037197768133911964, + "grad_norm": 1.2317055463790894, + "learning_rate": 0.0004990709685643932, + "loss": 1.9319, + "step": 360 + }, + { + "epoch": 0.038231039470965074, + "grad_norm": 1.074246883392334, + "learning_rate": 0.0004989990247318954, + "loss": 1.9447, + "step": 370 + }, + { + "epoch": 0.039264310808018184, + "grad_norm": 1.2657012939453125, + "learning_rate": 0.0004989244037332671, + "loss": 1.7053, + "step": 380 + }, + { + "epoch": 0.040297582145071294, + "grad_norm": 1.3143869638442993, + "learning_rate": 0.0004988471063708116, + "loss": 1.8009, + "step": 390 + }, + { + "epoch": 0.041330853482124404, + "grad_norm": 1.381238579750061, + "learning_rate": 0.0004987671334756084, + "loss": 1.8229, + "step": 400 + }, + { + "epoch": 0.042364124819177514, + "grad_norm": 1.2067453861236572, + "learning_rate": 0.0004986844859075028, + "loss": 1.7401, + "step": 410 + }, + { + "epoch": 0.04339739615623062, + "grad_norm": 1.2656382322311401, + "learning_rate": 0.0004985991645550979, + "loss": 1.6774, + "step": 420 + }, + { + "epoch": 0.04443066749328373, + "grad_norm": 0.9306573271751404, + "learning_rate": 0.0004985111703357444, + "loss": 1.7884, + "step": 430 + }, + { + "epoch": 0.04546393883033684, + "grad_norm": 0.9404155611991882, + "learning_rate": 0.0004984205041955309, + "loss": 1.6522, + "step": 440 + }, + { + "epoch": 0.04649721016738995, + "grad_norm": 1.2634170055389404, + "learning_rate": 0.0004983271671092738, + "loss": 1.7681, + "step": 450 + }, + { + "epoch": 0.04753048150444307, + "grad_norm": 1.6461280584335327, + "learning_rate": 0.0004982311600805066, + "loss": 1.8906, + "step": 460 + }, + { + "epoch": 0.04856375284149618, + "grad_norm": 1.242746114730835, + "learning_rate": 0.0004981324841414695, + "loss": 1.8049, + "step": 470 + }, + { + "epoch": 0.04959702417854929, + "grad_norm": 1.0817033052444458, + "learning_rate": 0.0004980311403530978, + "loss": 1.6272, + "step": 480 + }, + { + "epoch": 0.0506302955156024, + "grad_norm": 1.2890691757202148, + "learning_rate": 0.0004979271298050107, + "loss": 1.6988, + "step": 490 + }, + { + "epoch": 0.05166356685265551, + "grad_norm": 1.0929986238479614, + "learning_rate": 0.0004978204536155003, + "loss": 1.728, + "step": 500 + }, + { + "epoch": 0.05269683818970862, + "grad_norm": 1.2483139038085938, + "learning_rate": 0.0004977111129315177, + "loss": 1.5616, + "step": 510 + }, + { + "epoch": 0.05373010952676173, + "grad_norm": 1.0845454931259155, + "learning_rate": 0.0004975991089286632, + "loss": 1.5409, + "step": 520 + }, + { + "epoch": 0.05476338086381484, + "grad_norm": 1.4180911779403687, + "learning_rate": 0.0004974844428111715, + "loss": 1.7888, + "step": 530 + }, + { + "epoch": 0.05579665220086795, + "grad_norm": 1.0860427618026733, + "learning_rate": 0.0004973671158118998, + "loss": 1.6986, + "step": 540 + }, + { + "epoch": 0.05682992353792106, + "grad_norm": 1.1166560649871826, + "learning_rate": 0.0004972471291923143, + "loss": 1.8097, + "step": 550 + }, + { + "epoch": 0.05786319487497417, + "grad_norm": 1.2548375129699707, + "learning_rate": 0.0004971244842424768, + "loss": 1.6169, + "step": 560 + }, + { + "epoch": 0.05889646621202728, + "grad_norm": 0.991226851940155, + "learning_rate": 0.0004969991822810307, + "loss": 1.6965, + "step": 570 + }, + { + "epoch": 0.05992973754908039, + "grad_norm": 1.1663395166397095, + "learning_rate": 0.0004968712246551868, + "loss": 1.6587, + "step": 580 + }, + { + "epoch": 0.0609630088861335, + "grad_norm": 1.3420109748840332, + "learning_rate": 0.0004967406127407086, + "loss": 1.6837, + "step": 590 + }, + { + "epoch": 0.06199628022318661, + "grad_norm": 1.0421655178070068, + "learning_rate": 0.0004966073479418982, + "loss": 1.637, + "step": 600 + }, + { + "epoch": 0.06302955156023972, + "grad_norm": 1.2786924839019775, + "learning_rate": 0.0004964714316915803, + "loss": 1.5727, + "step": 610 + }, + { + "epoch": 0.06406282289729283, + "grad_norm": 1.507780909538269, + "learning_rate": 0.0004963328654510877, + "loss": 1.6056, + "step": 620 + }, + { + "epoch": 0.06509609423434594, + "grad_norm": 1.0936928987503052, + "learning_rate": 0.0004961916507102447, + "loss": 1.5742, + "step": 630 + }, + { + "epoch": 0.06612936557139905, + "grad_norm": 1.2347360849380493, + "learning_rate": 0.0004960477889873517, + "loss": 1.6692, + "step": 640 + }, + { + "epoch": 0.06716263690845216, + "grad_norm": 1.4181699752807617, + "learning_rate": 0.0004959012818291688, + "loss": 1.6412, + "step": 650 + }, + { + "epoch": 0.06819590824550527, + "grad_norm": 1.3150395154953003, + "learning_rate": 0.000495752130810899, + "loss": 1.4951, + "step": 660 + }, + { + "epoch": 0.06922917958255838, + "grad_norm": 1.5858718156814575, + "learning_rate": 0.0004956003375361712, + "loss": 1.6635, + "step": 670 + }, + { + "epoch": 0.07026245091961149, + "grad_norm": 1.1488349437713623, + "learning_rate": 0.0004954459036370231, + "loss": 1.6101, + "step": 680 + }, + { + "epoch": 0.0712957222566646, + "grad_norm": 1.0433565378189087, + "learning_rate": 0.0004952888307738839, + "loss": 1.7019, + "step": 690 + }, + { + "epoch": 0.07232899359371771, + "grad_norm": 1.2093909978866577, + "learning_rate": 0.000495129120635556, + "loss": 1.6477, + "step": 700 + }, + { + "epoch": 0.07336226493077082, + "grad_norm": 1.1268610954284668, + "learning_rate": 0.0004949667749391967, + "loss": 1.5262, + "step": 710 + }, + { + "epoch": 0.07439553626782393, + "grad_norm": 1.0410875082015991, + "learning_rate": 0.0004948017954303007, + "loss": 1.7201, + "step": 720 + }, + { + "epoch": 0.07542880760487704, + "grad_norm": 1.5368582010269165, + "learning_rate": 0.0004946341838826803, + "loss": 1.7024, + "step": 730 + }, + { + "epoch": 0.07646207894193015, + "grad_norm": 0.9913454651832581, + "learning_rate": 0.0004944639420984469, + "loss": 1.4697, + "step": 740 + }, + { + "epoch": 0.07749535027898326, + "grad_norm": 1.1712089776992798, + "learning_rate": 0.0004942910719079912, + "loss": 1.4866, + "step": 750 + }, + { + "epoch": 0.07852862161603637, + "grad_norm": 1.5118815898895264, + "learning_rate": 0.0004941155751699639, + "loss": 1.6489, + "step": 760 + }, + { + "epoch": 0.07956189295308948, + "grad_norm": 0.9319295883178711, + "learning_rate": 0.0004939374537712558, + "loss": 1.5053, + "step": 770 + }, + { + "epoch": 0.08059516429014259, + "grad_norm": 2.0669829845428467, + "learning_rate": 0.0004937567096269769, + "loss": 1.6412, + "step": 780 + }, + { + "epoch": 0.0816284356271957, + "grad_norm": 1.876446008682251, + "learning_rate": 0.0004935733446804367, + "loss": 1.7186, + "step": 790 + }, + { + "epoch": 0.08266170696424881, + "grad_norm": 1.0011149644851685, + "learning_rate": 0.0004933873609031224, + "loss": 1.4197, + "step": 800 + }, + { + "epoch": 0.08369497830130192, + "grad_norm": 84.56254577636719, + "learning_rate": 0.000493198760294678, + "loss": 2.006, + "step": 810 + }, + { + "epoch": 0.08472824963835503, + "grad_norm": 9.466862678527832, + "learning_rate": 0.0004930075448828836, + "loss": 3.6223, + "step": 820 + }, + { + "epoch": 0.08576152097540814, + "grad_norm": 1.8478171825408936, + "learning_rate": 0.0004928137167236323, + "loss": 1.8625, + "step": 830 + }, + { + "epoch": 0.08679479231246125, + "grad_norm": 1.6235121488571167, + "learning_rate": 0.0004926172779009089, + "loss": 1.6312, + "step": 840 + }, + { + "epoch": 0.08782806364951436, + "grad_norm": 1.0582070350646973, + "learning_rate": 0.0004924182305267672, + "loss": 1.4884, + "step": 850 + }, + { + "epoch": 0.08886133498656747, + "grad_norm": 1.6170542240142822, + "learning_rate": 0.0004922165767413077, + "loss": 1.4772, + "step": 860 + }, + { + "epoch": 0.08989460632362058, + "grad_norm": 1.1165497303009033, + "learning_rate": 0.0004920123187126539, + "loss": 1.396, + "step": 870 + }, + { + "epoch": 0.09092787766067369, + "grad_norm": 1.5052683353424072, + "learning_rate": 0.0004918054586369299, + "loss": 1.7157, + "step": 880 + }, + { + "epoch": 0.0919611489977268, + "grad_norm": 1.5653349161148071, + "learning_rate": 0.0004915959987382355, + "loss": 1.5859, + "step": 890 + }, + { + "epoch": 0.0929944203347799, + "grad_norm": 1.4743231534957886, + "learning_rate": 0.0004913839412686238, + "loss": 1.5155, + "step": 900 + }, + { + "epoch": 0.09402769167183303, + "grad_norm": 1.931356430053711, + "learning_rate": 0.0004911692885080757, + "loss": 1.6259, + "step": 910 + }, + { + "epoch": 0.09506096300888614, + "grad_norm": 1.209402322769165, + "learning_rate": 0.0004909520427644762, + "loss": 1.544, + "step": 920 + }, + { + "epoch": 0.09609423434593925, + "grad_norm": 1.3506582975387573, + "learning_rate": 0.0004907322063735895, + "loss": 1.6404, + "step": 930 + }, + { + "epoch": 0.09712750568299236, + "grad_norm": 1.0187022686004639, + "learning_rate": 0.0004905097816990331, + "loss": 1.4471, + "step": 940 + }, + { + "epoch": 0.09816077702004547, + "grad_norm": 1.3293033838272095, + "learning_rate": 0.0004902847711322535, + "loss": 1.5233, + "step": 950 + }, + { + "epoch": 0.09919404835709858, + "grad_norm": 1.0673298835754395, + "learning_rate": 0.0004900571770924999, + "loss": 1.4875, + "step": 960 + }, + { + "epoch": 0.10022731969415169, + "grad_norm": 1.2059558629989624, + "learning_rate": 0.000489827002026798, + "loss": 1.5143, + "step": 970 + }, + { + "epoch": 0.1012605910312048, + "grad_norm": 1.5264915227890015, + "learning_rate": 0.0004895942484099241, + "loss": 1.3904, + "step": 980 + }, + { + "epoch": 0.10229386236825791, + "grad_norm": 1.055259108543396, + "learning_rate": 0.0004893589187443786, + "loss": 1.5727, + "step": 990 + }, + { + "epoch": 0.10332713370531102, + "grad_norm": 1.3388723134994507, + "learning_rate": 0.0004891210155603585, + "loss": 1.3997, + "step": 1000 + }, + { + "epoch": 0.10436040504236413, + "grad_norm": 1.146324872970581, + "learning_rate": 0.0004888805414157304, + "loss": 1.5489, + "step": 1010 + }, + { + "epoch": 0.10539367637941724, + "grad_norm": 1.446535348892212, + "learning_rate": 0.0004886374988960036, + "loss": 1.5821, + "step": 1020 + }, + { + "epoch": 0.10642694771647035, + "grad_norm": 1.3649553060531616, + "learning_rate": 0.0004883918906143016, + "loss": 1.4677, + "step": 1030 + }, + { + "epoch": 0.10746021905352346, + "grad_norm": 1.2442352771759033, + "learning_rate": 0.00048814371921133417, + "loss": 1.4691, + "step": 1040 + }, + { + "epoch": 0.10849349039057657, + "grad_norm": 1.2473433017730713, + "learning_rate": 0.00048789298735536904, + "loss": 1.6451, + "step": 1050 + }, + { + "epoch": 0.10952676172762968, + "grad_norm": 1.7001152038574219, + "learning_rate": 0.0004876396977422033, + "loss": 1.6799, + "step": 1060 + }, + { + "epoch": 0.11056003306468279, + "grad_norm": 1.1802128553390503, + "learning_rate": 0.00048738385309513434, + "loss": 1.4816, + "step": 1070 + }, + { + "epoch": 0.1115933044017359, + "grad_norm": 1.1580235958099365, + "learning_rate": 0.0004871254561649303, + "loss": 1.681, + "step": 1080 + }, + { + "epoch": 0.11262657573878901, + "grad_norm": 1.2179391384124756, + "learning_rate": 0.000486864509729801, + "loss": 1.507, + "step": 1090 + }, + { + "epoch": 0.11365984707584212, + "grad_norm": 1.1689331531524658, + "learning_rate": 0.00048660101659536763, + "loss": 1.276, + "step": 1100 + }, + { + "epoch": 0.11469311841289523, + "grad_norm": 1.1789075136184692, + "learning_rate": 0.0004863349795946329, + "loss": 1.5065, + "step": 1110 + }, + { + "epoch": 0.11572638974994834, + "grad_norm": 0.98433917760849, + "learning_rate": 0.00048606640158795034, + "loss": 1.286, + "step": 1120 + }, + { + "epoch": 0.11675966108700145, + "grad_norm": 0.9510318636894226, + "learning_rate": 0.0004857952854629938, + "loss": 1.4637, + "step": 1130 + }, + { + "epoch": 0.11779293242405456, + "grad_norm": 1.0436633825302124, + "learning_rate": 0.0004855216341347259, + "loss": 1.4925, + "step": 1140 + }, + { + "epoch": 0.11882620376110767, + "grad_norm": 1.4930212497711182, + "learning_rate": 0.0004852454505453674, + "loss": 1.3586, + "step": 1150 + }, + { + "epoch": 0.11985947509816078, + "grad_norm": 0.8224968910217285, + "learning_rate": 0.00048496673766436517, + "loss": 1.5212, + "step": 1160 + }, + { + "epoch": 0.12089274643521389, + "grad_norm": 1.2815650701522827, + "learning_rate": 0.00048468549848835996, + "loss": 1.4901, + "step": 1170 + }, + { + "epoch": 0.121926017772267, + "grad_norm": 1.2067404985427856, + "learning_rate": 0.000484401736041155, + "loss": 1.5946, + "step": 1180 + }, + { + "epoch": 0.12295928910932011, + "grad_norm": 1.530190348625183, + "learning_rate": 0.0004841154533736827, + "loss": 1.5052, + "step": 1190 + }, + { + "epoch": 0.12399256044637322, + "grad_norm": 1.2579395771026611, + "learning_rate": 0.0004838266535639722, + "loss": 1.4186, + "step": 1200 + }, + { + "epoch": 0.12502583178342633, + "grad_norm": 1.5573033094406128, + "learning_rate": 0.00048353533971711625, + "loss": 1.3459, + "step": 1210 + }, + { + "epoch": 0.12605910312047944, + "grad_norm": 1.2861477136611938, + "learning_rate": 0.0004832415149652378, + "loss": 1.4907, + "step": 1220 + }, + { + "epoch": 0.12709237445753255, + "grad_norm": 0.8996832370758057, + "learning_rate": 0.00048294518246745643, + "loss": 1.3579, + "step": 1230 + }, + { + "epoch": 0.12812564579458566, + "grad_norm": 1.4587196111679077, + "learning_rate": 0.0004826463454098542, + "loss": 1.3771, + "step": 1240 + }, + { + "epoch": 0.12915891713163877, + "grad_norm": 1.1128901243209839, + "learning_rate": 0.00048234500700544144, + "loss": 1.3366, + "step": 1250 + }, + { + "epoch": 0.13019218846869188, + "grad_norm": 1.0845853090286255, + "learning_rate": 0.0004820411704941223, + "loss": 1.3485, + "step": 1260 + }, + { + "epoch": 0.131225459805745, + "grad_norm": 1.1594990491867065, + "learning_rate": 0.00048173483914265976, + "loss": 1.4484, + "step": 1270 + }, + { + "epoch": 0.1322587311427981, + "grad_norm": 1.2621511220932007, + "learning_rate": 0.0004814260162446408, + "loss": 1.4147, + "step": 1280 + }, + { + "epoch": 0.1332920024798512, + "grad_norm": 1.6021978855133057, + "learning_rate": 0.00048111470512044065, + "loss": 1.3724, + "step": 1290 + }, + { + "epoch": 0.13432527381690432, + "grad_norm": 1.3557963371276855, + "learning_rate": 0.0004808009091171873, + "loss": 1.3844, + "step": 1300 + }, + { + "epoch": 0.13535854515395743, + "grad_norm": 1.341562271118164, + "learning_rate": 0.0004804846316087254, + "loss": 1.2895, + "step": 1310 + }, + { + "epoch": 0.13639181649101054, + "grad_norm": 1.1911447048187256, + "learning_rate": 0.00048016587599557996, + "loss": 1.3963, + "step": 1320 + }, + { + "epoch": 0.13742508782806365, + "grad_norm": 1.0620297193527222, + "learning_rate": 0.0004798446457049201, + "loss": 1.3621, + "step": 1330 + }, + { + "epoch": 0.13845835916511676, + "grad_norm": 1.1608030796051025, + "learning_rate": 0.0004795209441905217, + "loss": 1.3658, + "step": 1340 + }, + { + "epoch": 0.13949163050216987, + "grad_norm": 1.5855592489242554, + "learning_rate": 0.0004791947749327308, + "loss": 1.3683, + "step": 1350 + }, + { + "epoch": 0.14052490183922298, + "grad_norm": 1.5676512718200684, + "learning_rate": 0.0004788661414384255, + "loss": 1.3186, + "step": 1360 + }, + { + "epoch": 0.1415581731762761, + "grad_norm": 1.149056315422058, + "learning_rate": 0.0004785350472409792, + "loss": 1.3231, + "step": 1370 + }, + { + "epoch": 0.1425914445133292, + "grad_norm": 0.9153057336807251, + "learning_rate": 0.00047820149590022153, + "loss": 1.2976, + "step": 1380 + }, + { + "epoch": 0.1436247158503823, + "grad_norm": 1.2601430416107178, + "learning_rate": 0.00047786549100240107, + "loss": 1.4205, + "step": 1390 + }, + { + "epoch": 0.14465798718743542, + "grad_norm": 1.2112687826156616, + "learning_rate": 0.0004775270361601461, + "loss": 1.3192, + "step": 1400 + }, + { + "epoch": 0.14569125852448853, + "grad_norm": 1.1192986965179443, + "learning_rate": 0.00047718613501242604, + "loss": 1.355, + "step": 1410 + }, + { + "epoch": 0.14672452986154164, + "grad_norm": 1.2099722623825073, + "learning_rate": 0.00047684279122451223, + "loss": 1.3638, + "step": 1420 + }, + { + "epoch": 0.14775780119859475, + "grad_norm": 1.124934434890747, + "learning_rate": 0.00047649700848793886, + "loss": 1.331, + "step": 1430 + }, + { + "epoch": 0.14879107253564786, + "grad_norm": 1.2407630681991577, + "learning_rate": 0.00047614879052046254, + "loss": 1.5308, + "step": 1440 + }, + { + "epoch": 0.14982434387270097, + "grad_norm": 1.2991951704025269, + "learning_rate": 0.00047579814106602316, + "loss": 1.3365, + "step": 1450 + }, + { + "epoch": 0.15085761520975408, + "grad_norm": 1.3195995092391968, + "learning_rate": 0.0004754450638947032, + "loss": 1.3209, + "step": 1460 + }, + { + "epoch": 0.15189088654680719, + "grad_norm": 1.4838430881500244, + "learning_rate": 0.000475089562802687, + "loss": 1.4705, + "step": 1470 + }, + { + "epoch": 0.1529241578838603, + "grad_norm": 1.126265287399292, + "learning_rate": 0.0004747316416122205, + "loss": 1.3316, + "step": 1480 + }, + { + "epoch": 0.1539574292209134, + "grad_norm": 1.0109881162643433, + "learning_rate": 0.00047437130417156973, + "loss": 1.4045, + "step": 1490 + }, + { + "epoch": 0.15499070055796652, + "grad_norm": 0.8355922698974609, + "learning_rate": 0.00047400855435497945, + "loss": 1.3058, + "step": 1500 + }, + { + "epoch": 0.15602397189501963, + "grad_norm": 1.406782865524292, + "learning_rate": 0.00047364339606263185, + "loss": 1.2732, + "step": 1510 + }, + { + "epoch": 0.15705724323207274, + "grad_norm": 0.9180629849433899, + "learning_rate": 0.00047327583322060406, + "loss": 1.3472, + "step": 1520 + }, + { + "epoch": 0.15809051456912585, + "grad_norm": 1.1642565727233887, + "learning_rate": 0.00047290586978082654, + "loss": 1.4165, + "step": 1530 + }, + { + "epoch": 0.15912378590617896, + "grad_norm": 1.6167621612548828, + "learning_rate": 0.00047253350972104003, + "loss": 1.5038, + "step": 1540 + }, + { + "epoch": 0.16015705724323206, + "grad_norm": 1.1963645219802856, + "learning_rate": 0.00047215875704475314, + "loss": 1.3744, + "step": 1550 + }, + { + "epoch": 0.16119032858028517, + "grad_norm": 1.246496319770813, + "learning_rate": 0.00047178161578119925, + "loss": 1.3314, + "step": 1560 + }, + { + "epoch": 0.16222359991733828, + "grad_norm": 1.1050803661346436, + "learning_rate": 0.00047140208998529303, + "loss": 1.4149, + "step": 1570 + }, + { + "epoch": 0.1632568712543914, + "grad_norm": 1.379414439201355, + "learning_rate": 0.00047102018373758716, + "loss": 1.3817, + "step": 1580 + }, + { + "epoch": 0.1642901425914445, + "grad_norm": 1.0111711025238037, + "learning_rate": 0.00047063590114422804, + "loss": 1.2865, + "step": 1590 + }, + { + "epoch": 0.16532341392849761, + "grad_norm": 1.5682810544967651, + "learning_rate": 0.00047024924633691197, + "loss": 1.317, + "step": 1600 + }, + { + "epoch": 0.16635668526555072, + "grad_norm": 0.9005609154701233, + "learning_rate": 0.0004698602234728406, + "loss": 1.3659, + "step": 1610 + }, + { + "epoch": 0.16738995660260383, + "grad_norm": 1.00556218624115, + "learning_rate": 0.00046946883673467624, + "loss": 1.3373, + "step": 1620 + }, + { + "epoch": 0.16842322793965694, + "grad_norm": 1.387304425239563, + "learning_rate": 0.00046907509033049683, + "loss": 1.4858, + "step": 1630 + }, + { + "epoch": 0.16945649927671005, + "grad_norm": 1.2545139789581299, + "learning_rate": 0.0004686789884937509, + "loss": 1.4221, + "step": 1640 + }, + { + "epoch": 0.17048977061376316, + "grad_norm": 1.3092625141143799, + "learning_rate": 0.00046828053548321187, + "loss": 1.3609, + "step": 1650 + }, + { + "epoch": 0.17152304195081627, + "grad_norm": 1.0507373809814453, + "learning_rate": 0.0004678797355829322, + "loss": 1.3406, + "step": 1660 + }, + { + "epoch": 0.17255631328786938, + "grad_norm": 1.0271055698394775, + "learning_rate": 0.0004674765931021976, + "loss": 1.3098, + "step": 1670 + }, + { + "epoch": 0.1735895846249225, + "grad_norm": 1.1777933835983276, + "learning_rate": 0.0004670711123754804, + "loss": 1.2113, + "step": 1680 + }, + { + "epoch": 0.1746228559619756, + "grad_norm": 1.1888736486434937, + "learning_rate": 0.00046666329776239306, + "loss": 1.3137, + "step": 1690 + }, + { + "epoch": 0.1756561272990287, + "grad_norm": 1.4923595190048218, + "learning_rate": 0.00046625315364764156, + "loss": 1.23, + "step": 1700 + }, + { + "epoch": 0.17668939863608182, + "grad_norm": 1.3654290437698364, + "learning_rate": 0.00046584068444097766, + "loss": 1.3182, + "step": 1710 + }, + { + "epoch": 0.17772266997313493, + "grad_norm": 1.5486831665039062, + "learning_rate": 0.0004654258945771521, + "loss": 1.2854, + "step": 1720 + }, + { + "epoch": 0.17875594131018804, + "grad_norm": 1.3204468488693237, + "learning_rate": 0.0004650087885158667, + "loss": 1.3328, + "step": 1730 + }, + { + "epoch": 0.17978921264724115, + "grad_norm": 1.4773411750793457, + "learning_rate": 0.000464589370741726, + "loss": 1.3741, + "step": 1740 + }, + { + "epoch": 0.18082248398429426, + "grad_norm": 1.20607328414917, + "learning_rate": 0.00046416764576418993, + "loss": 1.26, + "step": 1750 + }, + { + "epoch": 0.18185575532134737, + "grad_norm": 1.2402129173278809, + "learning_rate": 0.00046374361811752436, + "loss": 1.2398, + "step": 1760 + }, + { + "epoch": 0.18288902665840048, + "grad_norm": 1.1439170837402344, + "learning_rate": 0.00046331729236075305, + "loss": 1.4529, + "step": 1770 + }, + { + "epoch": 0.1839222979954536, + "grad_norm": 1.268829107284546, + "learning_rate": 0.0004628886730776084, + "loss": 1.4535, + "step": 1780 + }, + { + "epoch": 0.1849555693325067, + "grad_norm": 1.37249755859375, + "learning_rate": 0.0004624577648764819, + "loss": 1.3249, + "step": 1790 + }, + { + "epoch": 0.1859888406695598, + "grad_norm": 1.2565507888793945, + "learning_rate": 0.0004620245723903751, + "loss": 1.2866, + "step": 1800 + }, + { + "epoch": 0.18702211200661292, + "grad_norm": 1.1188476085662842, + "learning_rate": 0.0004615891002768494, + "loss": 1.1712, + "step": 1810 + }, + { + "epoch": 0.18805538334366606, + "grad_norm": 1.3613345623016357, + "learning_rate": 0.00046115135321797617, + "loss": 1.1788, + "step": 1820 + }, + { + "epoch": 0.18908865468071917, + "grad_norm": 1.3465733528137207, + "learning_rate": 0.00046071133592028626, + "loss": 1.2391, + "step": 1830 + }, + { + "epoch": 0.19012192601777228, + "grad_norm": 1.2718288898468018, + "learning_rate": 0.00046026905311471946, + "loss": 1.2213, + "step": 1840 + }, + { + "epoch": 0.1911551973548254, + "grad_norm": 1.372878909111023, + "learning_rate": 0.0004598245095565738, + "loss": 1.2036, + "step": 1850 + }, + { + "epoch": 0.1921884686918785, + "grad_norm": 1.198723316192627, + "learning_rate": 0.00045937771002545403, + "loss": 1.192, + "step": 1860 + }, + { + "epoch": 0.1932217400289316, + "grad_norm": 0.9199055433273315, + "learning_rate": 0.00045892865932522077, + "loss": 1.2999, + "step": 1870 + }, + { + "epoch": 0.19425501136598472, + "grad_norm": 1.3330020904541016, + "learning_rate": 0.0004584773622839383, + "loss": 1.3587, + "step": 1880 + }, + { + "epoch": 0.19528828270303783, + "grad_norm": 1.228247046470642, + "learning_rate": 0.0004580238237538232, + "loss": 1.4654, + "step": 1890 + }, + { + "epoch": 0.19632155404009094, + "grad_norm": 1.6616053581237793, + "learning_rate": 0.0004575680486111915, + "loss": 1.1583, + "step": 1900 + }, + { + "epoch": 0.19735482537714405, + "grad_norm": 1.0149476528167725, + "learning_rate": 0.000457110041756407, + "loss": 1.2121, + "step": 1910 + }, + { + "epoch": 0.19838809671419716, + "grad_norm": 1.6001296043395996, + "learning_rate": 0.00045664980811382813, + "loss": 1.2938, + "step": 1920 + }, + { + "epoch": 0.19942136805125027, + "grad_norm": 1.263951301574707, + "learning_rate": 0.00045618735263175495, + "loss": 1.1885, + "step": 1930 + }, + { + "epoch": 0.20045463938830338, + "grad_norm": 1.4672034978866577, + "learning_rate": 0.00045572268028237637, + "loss": 1.2981, + "step": 1940 + }, + { + "epoch": 0.2014879107253565, + "grad_norm": 1.1751412153244019, + "learning_rate": 0.0004552557960617161, + "loss": 1.117, + "step": 1950 + }, + { + "epoch": 0.2025211820624096, + "grad_norm": 1.131339192390442, + "learning_rate": 0.0004547867049895796, + "loss": 1.1909, + "step": 1960 + }, + { + "epoch": 0.2035544533994627, + "grad_norm": 0.991729736328125, + "learning_rate": 0.0004543154121094996, + "loss": 1.1328, + "step": 1970 + }, + { + "epoch": 0.20458772473651582, + "grad_norm": 0.9073534607887268, + "learning_rate": 0.0004538419224886819, + "loss": 0.9973, + "step": 1980 + }, + { + "epoch": 0.20562099607356893, + "grad_norm": 1.2480231523513794, + "learning_rate": 0.00045336624121795124, + "loss": 1.2367, + "step": 1990 + }, + { + "epoch": 0.20665426741062204, + "grad_norm": 1.28248131275177, + "learning_rate": 0.0004528883734116963, + "loss": 1.1738, + "step": 2000 + }, + { + "epoch": 0.20768753874767515, + "grad_norm": 1.386885166168213, + "learning_rate": 0.00045240832420781474, + "loss": 1.3788, + "step": 2010 + }, + { + "epoch": 0.20872081008472826, + "grad_norm": 1.1158126592636108, + "learning_rate": 0.0004519260987676579, + "loss": 1.3046, + "step": 2020 + }, + { + "epoch": 0.20975408142178137, + "grad_norm": 1.162636399269104, + "learning_rate": 0.00045144170227597545, + "loss": 1.2355, + "step": 2030 + }, + { + "epoch": 0.21078735275883448, + "grad_norm": 1.4077121019363403, + "learning_rate": 0.0004509551399408598, + "loss": 1.1542, + "step": 2040 + }, + { + "epoch": 0.2118206240958876, + "grad_norm": 1.1237014532089233, + "learning_rate": 0.00045046641699368953, + "loss": 1.3411, + "step": 2050 + }, + { + "epoch": 0.2128538954329407, + "grad_norm": 1.1503592729568481, + "learning_rate": 0.0004499755386890736, + "loss": 1.1783, + "step": 2060 + }, + { + "epoch": 0.2138871667699938, + "grad_norm": 1.2852717638015747, + "learning_rate": 0.0004494825103047949, + "loss": 1.3331, + "step": 2070 + }, + { + "epoch": 0.21492043810704692, + "grad_norm": 1.2611557245254517, + "learning_rate": 0.0004489873371417534, + "loss": 1.3272, + "step": 2080 + }, + { + "epoch": 0.21595370944410003, + "grad_norm": 1.4016764163970947, + "learning_rate": 0.00044849002452390874, + "loss": 1.1658, + "step": 2090 + }, + { + "epoch": 0.21698698078115314, + "grad_norm": 1.4920654296875, + "learning_rate": 0.0004479905777982238, + "loss": 1.2651, + "step": 2100 + }, + { + "epoch": 0.21802025211820625, + "grad_norm": 1.1262540817260742, + "learning_rate": 0.0004474890023346066, + "loss": 1.2347, + "step": 2110 + }, + { + "epoch": 0.21905352345525936, + "grad_norm": 1.2998685836791992, + "learning_rate": 0.0004469853035258526, + "loss": 1.3167, + "step": 2120 + }, + { + "epoch": 0.22008679479231247, + "grad_norm": 1.0999696254730225, + "learning_rate": 0.0004464794867875871, + "loss": 1.1621, + "step": 2130 + }, + { + "epoch": 0.22112006612936558, + "grad_norm": 0.9801570177078247, + "learning_rate": 0.0004459715575582066, + "loss": 1.1631, + "step": 2140 + }, + { + "epoch": 0.2221533374664187, + "grad_norm": 1.161098837852478, + "learning_rate": 0.00044546152129882054, + "loss": 1.1256, + "step": 2150 + }, + { + "epoch": 0.2231866088034718, + "grad_norm": 1.2714890241622925, + "learning_rate": 0.0004449493834931927, + "loss": 1.1182, + "step": 2160 + }, + { + "epoch": 0.2242198801405249, + "grad_norm": 0.9561129808425903, + "learning_rate": 0.0004444351496476818, + "loss": 1.2051, + "step": 2170 + }, + { + "epoch": 0.22525315147757802, + "grad_norm": 1.3160920143127441, + "learning_rate": 0.0004439188252911828, + "loss": 1.2268, + "step": 2180 + }, + { + "epoch": 0.22628642281463113, + "grad_norm": 1.1478712558746338, + "learning_rate": 0.0004434004159750671, + "loss": 1.1928, + "step": 2190 + }, + { + "epoch": 0.22731969415168424, + "grad_norm": 1.2426934242248535, + "learning_rate": 0.0004428799272731231, + "loss": 1.1548, + "step": 2200 + }, + { + "epoch": 0.22835296548873735, + "grad_norm": 1.3587703704833984, + "learning_rate": 0.0004423573647814961, + "loss": 1.1371, + "step": 2210 + }, + { + "epoch": 0.22938623682579046, + "grad_norm": 1.0638786554336548, + "learning_rate": 0.0004418327341186282, + "loss": 1.1668, + "step": 2220 + }, + { + "epoch": 0.23041950816284357, + "grad_norm": 1.1600587368011475, + "learning_rate": 0.00044130604092519794, + "loss": 1.1524, + "step": 2230 + }, + { + "epoch": 0.23145277949989668, + "grad_norm": 1.402921199798584, + "learning_rate": 0.0004407772908640595, + "loss": 1.3963, + "step": 2240 + }, + { + "epoch": 0.2324860508369498, + "grad_norm": 1.3043389320373535, + "learning_rate": 0.00044024648962018184, + "loss": 1.2359, + "step": 2250 + }, + { + "epoch": 0.2335193221740029, + "grad_norm": 1.1807844638824463, + "learning_rate": 0.0004397136429005879, + "loss": 1.366, + "step": 2260 + }, + { + "epoch": 0.234552593511056, + "grad_norm": 1.397064208984375, + "learning_rate": 0.00043917875643429284, + "loss": 1.234, + "step": 2270 + }, + { + "epoch": 0.23558586484810912, + "grad_norm": 1.3520923852920532, + "learning_rate": 0.0004386418359722426, + "loss": 1.1863, + "step": 2280 + }, + { + "epoch": 0.23661913618516223, + "grad_norm": 1.3684437274932861, + "learning_rate": 0.00043810288728725203, + "loss": 1.3008, + "step": 2290 + }, + { + "epoch": 0.23765240752221534, + "grad_norm": 1.1893550157546997, + "learning_rate": 0.0004375619161739428, + "loss": 1.0295, + "step": 2300 + }, + { + "epoch": 0.23868567885926845, + "grad_norm": 1.57694673538208, + "learning_rate": 0.0004370189284486814, + "loss": 1.2778, + "step": 2310 + }, + { + "epoch": 0.23971895019632156, + "grad_norm": 1.601629376411438, + "learning_rate": 0.0004364739299495162, + "loss": 1.0583, + "step": 2320 + }, + { + "epoch": 0.24075222153337467, + "grad_norm": 1.3246372938156128, + "learning_rate": 0.00043592692653611465, + "loss": 1.201, + "step": 2330 + }, + { + "epoch": 0.24178549287042778, + "grad_norm": 1.020665168762207, + "learning_rate": 0.0004353779240897008, + "loss": 1.3127, + "step": 2340 + }, + { + "epoch": 0.24281876420748089, + "grad_norm": 1.082587480545044, + "learning_rate": 0.00043482692851299146, + "loss": 1.225, + "step": 2350 + }, + { + "epoch": 0.243852035544534, + "grad_norm": 1.093636155128479, + "learning_rate": 0.00043427394573013314, + "loss": 1.0587, + "step": 2360 + }, + { + "epoch": 0.2448853068815871, + "grad_norm": 1.3400568962097168, + "learning_rate": 0.00043371898168663816, + "loss": 1.2375, + "step": 2370 + }, + { + "epoch": 0.24591857821864022, + "grad_norm": 1.223877191543579, + "learning_rate": 0.00043316204234932084, + "loss": 1.3076, + "step": 2380 + }, + { + "epoch": 0.24695184955569333, + "grad_norm": 1.4696872234344482, + "learning_rate": 0.0004326031337062333, + "loss": 1.251, + "step": 2390 + }, + { + "epoch": 0.24798512089274644, + "grad_norm": 1.3883171081542969, + "learning_rate": 0.00043204226176660107, + "loss": 1.2215, + "step": 2400 + }, + { + "epoch": 0.24901839222979955, + "grad_norm": 1.1045054197311401, + "learning_rate": 0.0004314794325607584, + "loss": 1.1014, + "step": 2410 + }, + { + "epoch": 0.25005166356685266, + "grad_norm": 1.273431658744812, + "learning_rate": 0.00043091465214008354, + "loss": 1.0845, + "step": 2420 + }, + { + "epoch": 0.25108493490390577, + "grad_norm": 1.0618531703948975, + "learning_rate": 0.0004303479265769337, + "loss": 1.1162, + "step": 2430 + }, + { + "epoch": 0.2521182062409589, + "grad_norm": 1.3230130672454834, + "learning_rate": 0.0004297792619645797, + "loss": 1.0385, + "step": 2440 + }, + { + "epoch": 0.253151477578012, + "grad_norm": 1.3020095825195312, + "learning_rate": 0.0004292086644171403, + "loss": 1.1548, + "step": 2450 + }, + { + "epoch": 0.2541847489150651, + "grad_norm": 1.3359605073928833, + "learning_rate": 0.0004286361400695169, + "loss": 1.3534, + "step": 2460 + }, + { + "epoch": 0.2552180202521182, + "grad_norm": 0.9598666429519653, + "learning_rate": 0.00042806169507732706, + "loss": 1.2432, + "step": 2470 + }, + { + "epoch": 0.2562512915891713, + "grad_norm": 0.9668743014335632, + "learning_rate": 0.00042748533561683865, + "loss": 1.2512, + "step": 2480 + }, + { + "epoch": 0.2572845629262244, + "grad_norm": 1.0706520080566406, + "learning_rate": 0.0004269070678849034, + "loss": 1.136, + "step": 2490 + }, + { + "epoch": 0.25831783426327753, + "grad_norm": 1.0470647811889648, + "learning_rate": 0.00042632689809889027, + "loss": 1.1211, + "step": 2500 + }, + { + "epoch": 0.25935110560033064, + "grad_norm": 1.1314566135406494, + "learning_rate": 0.0004257448324966183, + "loss": 1.0704, + "step": 2510 + }, + { + "epoch": 0.26038437693738375, + "grad_norm": 1.085749626159668, + "learning_rate": 0.00042516087733629004, + "loss": 1.1727, + "step": 2520 + }, + { + "epoch": 0.26141764827443686, + "grad_norm": 1.2759054899215698, + "learning_rate": 0.00042457503889642396, + "loss": 1.2864, + "step": 2530 + }, + { + "epoch": 0.26245091961149, + "grad_norm": 1.2468546628952026, + "learning_rate": 0.0004239873234757871, + "loss": 1.0734, + "step": 2540 + }, + { + "epoch": 0.2634841909485431, + "grad_norm": 1.2701605558395386, + "learning_rate": 0.00042339773739332706, + "loss": 1.1816, + "step": 2550 + }, + { + "epoch": 0.2645174622855962, + "grad_norm": 1.0027897357940674, + "learning_rate": 0.00042280628698810447, + "loss": 1.0726, + "step": 2560 + }, + { + "epoch": 0.2655507336226493, + "grad_norm": 1.4462809562683105, + "learning_rate": 0.0004222129786192245, + "loss": 1.278, + "step": 2570 + }, + { + "epoch": 0.2665840049597024, + "grad_norm": 1.241769552230835, + "learning_rate": 0.0004216178186657686, + "loss": 1.1829, + "step": 2580 + }, + { + "epoch": 0.2676172762967555, + "grad_norm": 1.6216635704040527, + "learning_rate": 0.00042102081352672594, + "loss": 1.1479, + "step": 2590 + }, + { + "epoch": 0.26865054763380863, + "grad_norm": 4.33831787109375, + "learning_rate": 0.0004204219696209248, + "loss": 1.1709, + "step": 2600 + }, + { + "epoch": 0.26968381897086174, + "grad_norm": 0.952393114566803, + "learning_rate": 0.00041982129338696296, + "loss": 1.1018, + "step": 2610 + }, + { + "epoch": 0.27071709030791485, + "grad_norm": 1.4484752416610718, + "learning_rate": 0.0004192187912831393, + "loss": 1.081, + "step": 2620 + }, + { + "epoch": 0.27175036164496796, + "grad_norm": 1.0843290090560913, + "learning_rate": 0.0004186144697873835, + "loss": 1.251, + "step": 2630 + }, + { + "epoch": 0.2727836329820211, + "grad_norm": 1.5165774822235107, + "learning_rate": 0.0004180083353971871, + "loss": 1.134, + "step": 2640 + }, + { + "epoch": 0.2738169043190742, + "grad_norm": 1.2171909809112549, + "learning_rate": 0.00041740039462953343, + "loss": 1.1763, + "step": 2650 + }, + { + "epoch": 0.2748501756561273, + "grad_norm": 1.0561660528182983, + "learning_rate": 0.00041679065402082724, + "loss": 1.1178, + "step": 2660 + }, + { + "epoch": 0.2758834469931804, + "grad_norm": 1.2738356590270996, + "learning_rate": 0.0004161791201268247, + "loss": 1.2379, + "step": 2670 + }, + { + "epoch": 0.2769167183302335, + "grad_norm": 1.127266764640808, + "learning_rate": 0.000415565799522563, + "loss": 1.2347, + "step": 2680 + }, + { + "epoch": 0.2779499896672866, + "grad_norm": 1.4078705310821533, + "learning_rate": 0.0004149506988022894, + "loss": 1.2155, + "step": 2690 + }, + { + "epoch": 0.27898326100433973, + "grad_norm": 0.9871230125427246, + "learning_rate": 0.0004143338245793906, + "loss": 1.1583, + "step": 2700 + }, + { + "epoch": 0.28001653234139284, + "grad_norm": 0.8731614947319031, + "learning_rate": 0.0004137151834863213, + "loss": 1.171, + "step": 2710 + }, + { + "epoch": 0.28104980367844595, + "grad_norm": 1.0814101696014404, + "learning_rate": 0.0004130947821745332, + "loss": 1.2112, + "step": 2720 + }, + { + "epoch": 0.28208307501549906, + "grad_norm": 1.695854902267456, + "learning_rate": 0.0004124726273144034, + "loss": 1.0269, + "step": 2730 + }, + { + "epoch": 0.2831163463525522, + "grad_norm": 0.9004189372062683, + "learning_rate": 0.00041184872559516244, + "loss": 1.1974, + "step": 2740 + }, + { + "epoch": 0.2841496176896053, + "grad_norm": 1.034590482711792, + "learning_rate": 0.0004112230837248229, + "loss": 1.1068, + "step": 2750 + }, + { + "epoch": 0.2851828890266584, + "grad_norm": 1.4124444723129272, + "learning_rate": 0.0004105957084301069, + "loss": 1.0922, + "step": 2760 + }, + { + "epoch": 0.2862161603637115, + "grad_norm": 1.4991061687469482, + "learning_rate": 0.00040996660645637345, + "loss": 1.2392, + "step": 2770 + }, + { + "epoch": 0.2872494317007646, + "grad_norm": 1.316277027130127, + "learning_rate": 0.00040933578456754687, + "loss": 1.0433, + "step": 2780 + }, + { + "epoch": 0.2882827030378177, + "grad_norm": 1.6624524593353271, + "learning_rate": 0.00040870324954604323, + "loss": 0.9623, + "step": 2790 + }, + { + "epoch": 0.28931597437487083, + "grad_norm": 1.2658953666687012, + "learning_rate": 0.00040806900819269766, + "loss": 1.108, + "step": 2800 + }, + { + "epoch": 0.29034924571192394, + "grad_norm": 1.3295769691467285, + "learning_rate": 0.00040743306732669133, + "loss": 1.137, + "step": 2810 + }, + { + "epoch": 0.29138251704897705, + "grad_norm": 1.6526551246643066, + "learning_rate": 0.0004067954337854783, + "loss": 1.1281, + "step": 2820 + }, + { + "epoch": 0.29241578838603016, + "grad_norm": 1.1587738990783691, + "learning_rate": 0.0004061561144247115, + "loss": 1.0086, + "step": 2830 + }, + { + "epoch": 0.29344905972308327, + "grad_norm": 1.454256534576416, + "learning_rate": 0.0004055151161181694, + "loss": 1.0723, + "step": 2840 + }, + { + "epoch": 0.2944823310601364, + "grad_norm": 1.476521611213684, + "learning_rate": 0.00040487244575768193, + "loss": 1.2486, + "step": 2850 + }, + { + "epoch": 0.2955156023971895, + "grad_norm": 0.9193058609962463, + "learning_rate": 0.00040422811025305676, + "loss": 1.1071, + "step": 2860 + }, + { + "epoch": 0.2965488737342426, + "grad_norm": 1.8984912633895874, + "learning_rate": 0.0004035821165320045, + "loss": 1.1724, + "step": 2870 + }, + { + "epoch": 0.2975821450712957, + "grad_norm": 1.7232333421707153, + "learning_rate": 0.00040293447154006435, + "loss": 1.174, + "step": 2880 + }, + { + "epoch": 0.2986154164083488, + "grad_norm": 0.9074342846870422, + "learning_rate": 0.0004022851822405297, + "loss": 1.2448, + "step": 2890 + }, + { + "epoch": 0.29964868774540193, + "grad_norm": 1.2896414995193481, + "learning_rate": 0.0004016342556143727, + "loss": 1.1207, + "step": 2900 + }, + { + "epoch": 0.30068195908245504, + "grad_norm": 1.264952301979065, + "learning_rate": 0.0004009816986601702, + "loss": 1.0868, + "step": 2910 + }, + { + "epoch": 0.30171523041950815, + "grad_norm": 1.1345736980438232, + "learning_rate": 0.00040032751839402727, + "loss": 1.1881, + "step": 2920 + }, + { + "epoch": 0.30274850175656126, + "grad_norm": 1.0581121444702148, + "learning_rate": 0.00039967172184950263, + "loss": 1.1796, + "step": 2930 + }, + { + "epoch": 0.30378177309361437, + "grad_norm": 1.1877319812774658, + "learning_rate": 0.0003990143160775327, + "loss": 1.108, + "step": 2940 + }, + { + "epoch": 0.3048150444306675, + "grad_norm": 1.0478935241699219, + "learning_rate": 0.0003983553081463559, + "loss": 0.9986, + "step": 2950 + }, + { + "epoch": 0.3058483157677206, + "grad_norm": 1.1299808025360107, + "learning_rate": 0.0003976947051414366, + "loss": 1.0221, + "step": 2960 + }, + { + "epoch": 0.3068815871047737, + "grad_norm": 1.0618709325790405, + "learning_rate": 0.00039703251416538883, + "loss": 0.9024, + "step": 2970 + }, + { + "epoch": 0.3079148584418268, + "grad_norm": 1.099435806274414, + "learning_rate": 0.0003963687423379003, + "loss": 1.2754, + "step": 2980 + }, + { + "epoch": 0.3089481297788799, + "grad_norm": 1.1851657629013062, + "learning_rate": 0.00039570339679565527, + "loss": 1.1154, + "step": 2990 + }, + { + "epoch": 0.30998140111593303, + "grad_norm": 1.1812468767166138, + "learning_rate": 0.00039503648469225826, + "loss": 0.9979, + "step": 3000 + }, + { + "epoch": 0.31101467245298614, + "grad_norm": 1.1941215991973877, + "learning_rate": 0.00039436801319815717, + "loss": 1.0625, + "step": 3010 + }, + { + "epoch": 0.31204794379003925, + "grad_norm": 0.9969115853309631, + "learning_rate": 0.00039369798950056567, + "loss": 1.189, + "step": 3020 + }, + { + "epoch": 0.31308121512709236, + "grad_norm": 1.1364282369613647, + "learning_rate": 0.00039302642080338664, + "loss": 1.107, + "step": 3030 + }, + { + "epoch": 0.31411448646414547, + "grad_norm": 1.234078288078308, + "learning_rate": 0.0003923533143271341, + "loss": 1.238, + "step": 3040 + }, + { + "epoch": 0.3151477578011986, + "grad_norm": 1.0862330198287964, + "learning_rate": 0.0003916786773088559, + "loss": 1.1151, + "step": 3050 + }, + { + "epoch": 0.3161810291382517, + "grad_norm": 0.7923426032066345, + "learning_rate": 0.0003910025170020558, + "loss": 1.1031, + "step": 3060 + }, + { + "epoch": 0.3172143004753048, + "grad_norm": 1.6818230152130127, + "learning_rate": 0.0003903248406766158, + "loss": 1.0546, + "step": 3070 + }, + { + "epoch": 0.3182475718123579, + "grad_norm": 1.0749332904815674, + "learning_rate": 0.00038964565561871723, + "loss": 1.1412, + "step": 3080 + }, + { + "epoch": 0.319280843149411, + "grad_norm": 1.6896083354949951, + "learning_rate": 0.00038896496913076327, + "loss": 1.0668, + "step": 3090 + }, + { + "epoch": 0.32031411448646413, + "grad_norm": 1.24112069606781, + "learning_rate": 0.0003882827885312998, + "loss": 1.2632, + "step": 3100 + }, + { + "epoch": 0.32134738582351724, + "grad_norm": 1.2017937898635864, + "learning_rate": 0.00038759912115493724, + "loss": 1.0969, + "step": 3110 + }, + { + "epoch": 0.32238065716057035, + "grad_norm": 1.20210599899292, + "learning_rate": 0.00038691397435227097, + "loss": 0.9202, + "step": 3120 + }, + { + "epoch": 0.32341392849762346, + "grad_norm": 1.408257246017456, + "learning_rate": 0.0003862273554898032, + "loss": 1.1583, + "step": 3130 + }, + { + "epoch": 0.32444719983467657, + "grad_norm": 1.5274341106414795, + "learning_rate": 0.000385539271949863, + "loss": 1.0382, + "step": 3140 + }, + { + "epoch": 0.3254804711717297, + "grad_norm": 1.1086093187332153, + "learning_rate": 0.00038484973113052736, + "loss": 1.1232, + "step": 3150 + }, + { + "epoch": 0.3265137425087828, + "grad_norm": 1.35590660572052, + "learning_rate": 0.0003841587404455413, + "loss": 1.1019, + "step": 3160 + }, + { + "epoch": 0.3275470138458359, + "grad_norm": 1.0598667860031128, + "learning_rate": 0.000383466307324239, + "loss": 1.1557, + "step": 3170 + }, + { + "epoch": 0.328580285182889, + "grad_norm": 1.1116646528244019, + "learning_rate": 0.00038277243921146253, + "loss": 1.0634, + "step": 3180 + }, + { + "epoch": 0.3296135565199421, + "grad_norm": 1.4738123416900635, + "learning_rate": 0.0003820771435674829, + "loss": 1.0067, + "step": 3190 + }, + { + "epoch": 0.33064682785699523, + "grad_norm": 1.3569648265838623, + "learning_rate": 0.00038138042786791987, + "loss": 1.0798, + "step": 3200 + }, + { + "epoch": 0.33168009919404834, + "grad_norm": 1.073232650756836, + "learning_rate": 0.00038068229960366054, + "loss": 1.0349, + "step": 3210 + }, + { + "epoch": 0.33271337053110145, + "grad_norm": 1.4980833530426025, + "learning_rate": 0.0003799827662807801, + "loss": 1.1281, + "step": 3220 + }, + { + "epoch": 0.33374664186815456, + "grad_norm": 1.4308544397354126, + "learning_rate": 0.0003792818354204601, + "loss": 1.1056, + "step": 3230 + }, + { + "epoch": 0.33477991320520767, + "grad_norm": 1.5736769437789917, + "learning_rate": 0.0003785795145589085, + "loss": 1.0928, + "step": 3240 + }, + { + "epoch": 0.3358131845422608, + "grad_norm": 1.5231635570526123, + "learning_rate": 0.0003778758112472776, + "loss": 1.2857, + "step": 3250 + }, + { + "epoch": 0.3368464558793139, + "grad_norm": 1.0374634265899658, + "learning_rate": 0.00037717073305158376, + "loss": 1.1738, + "step": 3260 + }, + { + "epoch": 0.337879727216367, + "grad_norm": 1.1238751411437988, + "learning_rate": 0.0003764642875526256, + "loss": 0.9849, + "step": 3270 + }, + { + "epoch": 0.3389129985534201, + "grad_norm": 1.039632797241211, + "learning_rate": 0.0003757564823459025, + "loss": 1.059, + "step": 3280 + }, + { + "epoch": 0.3399462698904732, + "grad_norm": 1.1179864406585693, + "learning_rate": 0.0003750473250415334, + "loss": 0.9745, + "step": 3290 + }, + { + "epoch": 0.34097954122752633, + "grad_norm": 1.1467711925506592, + "learning_rate": 0.0003743368232641741, + "loss": 1.1077, + "step": 3300 + }, + { + "epoch": 0.34201281256457944, + "grad_norm": 1.0984638929367065, + "learning_rate": 0.00037362498465293604, + "loss": 1.0454, + "step": 3310 + }, + { + "epoch": 0.34304608390163255, + "grad_norm": 1.0687668323516846, + "learning_rate": 0.000372911816861304, + "loss": 1.073, + "step": 3320 + }, + { + "epoch": 0.34407935523868566, + "grad_norm": 1.6367201805114746, + "learning_rate": 0.00037219732755705353, + "loss": 1.1094, + "step": 3330 + }, + { + "epoch": 0.34511262657573877, + "grad_norm": 1.2905350923538208, + "learning_rate": 0.0003714815244221689, + "loss": 1.1536, + "step": 3340 + }, + { + "epoch": 0.3461458979127919, + "grad_norm": 0.9039587378501892, + "learning_rate": 0.00037076441515276003, + "loss": 1.1906, + "step": 3350 + }, + { + "epoch": 0.347179169249845, + "grad_norm": 1.3063287734985352, + "learning_rate": 0.0003700460074589804, + "loss": 1.1448, + "step": 3360 + }, + { + "epoch": 0.3482124405868981, + "grad_norm": 1.5665302276611328, + "learning_rate": 0.00036932630906494346, + "loss": 1.2537, + "step": 3370 + }, + { + "epoch": 0.3492457119239512, + "grad_norm": 1.0483524799346924, + "learning_rate": 0.0003686053277086401, + "loss": 1.1135, + "step": 3380 + }, + { + "epoch": 0.3502789832610043, + "grad_norm": 1.2478828430175781, + "learning_rate": 0.0003678830711418551, + "loss": 1.0816, + "step": 3390 + }, + { + "epoch": 0.3513122545980574, + "grad_norm": 1.14175283908844, + "learning_rate": 0.00036715954713008406, + "loss": 1.1217, + "step": 3400 + }, + { + "epoch": 0.35234552593511054, + "grad_norm": 1.0000718832015991, + "learning_rate": 0.0003664347634524497, + "loss": 1.1127, + "step": 3410 + }, + { + "epoch": 0.35337879727216365, + "grad_norm": 1.5337084531784058, + "learning_rate": 0.00036570872790161834, + "loss": 1.0619, + "step": 3420 + }, + { + "epoch": 0.35441206860921676, + "grad_norm": 0.878587543964386, + "learning_rate": 0.00036498144828371604, + "loss": 1.1094, + "step": 3430 + }, + { + "epoch": 0.35544533994626987, + "grad_norm": 1.3913145065307617, + "learning_rate": 0.0003642529324182449, + "loss": 0.9874, + "step": 3440 + }, + { + "epoch": 0.356478611283323, + "grad_norm": 1.388934850692749, + "learning_rate": 0.0003635231881379985, + "loss": 1.0177, + "step": 3450 + }, + { + "epoch": 0.3575118826203761, + "grad_norm": 1.2259199619293213, + "learning_rate": 0.00036279222328897826, + "loss": 1.0533, + "step": 3460 + }, + { + "epoch": 0.3585451539574292, + "grad_norm": 1.561023235321045, + "learning_rate": 0.0003620600457303085, + "loss": 1.1231, + "step": 3470 + }, + { + "epoch": 0.3595784252944823, + "grad_norm": 1.6612300872802734, + "learning_rate": 0.0003613266633341528, + "loss": 1.0971, + "step": 3480 + }, + { + "epoch": 0.3606116966315354, + "grad_norm": 1.3272353410720825, + "learning_rate": 0.00036059208398562793, + "loss": 1.024, + "step": 3490 + }, + { + "epoch": 0.3616449679685885, + "grad_norm": 1.2441381216049194, + "learning_rate": 0.00035985631558272074, + "loss": 1.0369, + "step": 3500 + }, + { + "epoch": 0.36267823930564164, + "grad_norm": 1.1805377006530762, + "learning_rate": 0.00035911936603620225, + "loss": 0.9626, + "step": 3510 + }, + { + "epoch": 0.36371151064269475, + "grad_norm": 1.3905142545700073, + "learning_rate": 0.00035838124326954254, + "loss": 1.1415, + "step": 3520 + }, + { + "epoch": 0.36474478197974786, + "grad_norm": 1.2646347284317017, + "learning_rate": 0.0003576419552188261, + "loss": 1.0101, + "step": 3530 + }, + { + "epoch": 0.36577805331680097, + "grad_norm": 1.1667832136154175, + "learning_rate": 0.000356901509832666, + "loss": 0.9862, + "step": 3540 + }, + { + "epoch": 0.3668113246538541, + "grad_norm": 0.8571506142616272, + "learning_rate": 0.00035615991507211887, + "loss": 0.9644, + "step": 3550 + }, + { + "epoch": 0.3678445959909072, + "grad_norm": 0.9799628853797913, + "learning_rate": 0.00035541717891059886, + "loss": 0.8918, + "step": 3560 + }, + { + "epoch": 0.3688778673279603, + "grad_norm": 1.0068583488464355, + "learning_rate": 0.0003546733093337924, + "loss": 0.935, + "step": 3570 + }, + { + "epoch": 0.3699111386650134, + "grad_norm": 0.8947041034698486, + "learning_rate": 0.0003539283143395719, + "loss": 1.0752, + "step": 3580 + }, + { + "epoch": 0.3709444100020665, + "grad_norm": 1.0797079801559448, + "learning_rate": 0.00035318220193790976, + "loss": 0.9534, + "step": 3590 + }, + { + "epoch": 0.3719776813391196, + "grad_norm": 1.0746815204620361, + "learning_rate": 0.0003524349801507929, + "loss": 0.9862, + "step": 3600 + }, + { + "epoch": 0.37301095267617274, + "grad_norm": 1.0419459342956543, + "learning_rate": 0.0003516866570121356, + "loss": 1.0606, + "step": 3610 + }, + { + "epoch": 0.37404422401322585, + "grad_norm": 1.1848483085632324, + "learning_rate": 0.0003509372405676937, + "loss": 0.97, + "step": 3620 + }, + { + "epoch": 0.375077495350279, + "grad_norm": 1.2240206003189087, + "learning_rate": 0.00035018673887497807, + "loss": 1.0237, + "step": 3630 + }, + { + "epoch": 0.3761107666873321, + "grad_norm": 1.6243517398834229, + "learning_rate": 0.0003494351600031677, + "loss": 1.0748, + "step": 3640 + }, + { + "epoch": 0.37714403802438523, + "grad_norm": 1.4265719652175903, + "learning_rate": 0.00034868251203302314, + "loss": 1.0639, + "step": 3650 + }, + { + "epoch": 0.37817730936143834, + "grad_norm": 1.0866336822509766, + "learning_rate": 0.00034792880305679967, + "loss": 0.9635, + "step": 3660 + }, + { + "epoch": 0.37921058069849145, + "grad_norm": 1.0087511539459229, + "learning_rate": 0.00034717404117816, + "loss": 1.1082, + "step": 3670 + }, + { + "epoch": 0.38024385203554456, + "grad_norm": 0.9332922697067261, + "learning_rate": 0.0003464182345120877, + "loss": 1.0258, + "step": 3680 + }, + { + "epoch": 0.38127712337259767, + "grad_norm": 0.9739323854446411, + "learning_rate": 0.000345661391184799, + "loss": 1.0168, + "step": 3690 + }, + { + "epoch": 0.3823103947096508, + "grad_norm": 0.912225067615509, + "learning_rate": 0.00034490351933365673, + "loss": 1.0996, + "step": 3700 + }, + { + "epoch": 0.3833436660467039, + "grad_norm": 0.853449285030365, + "learning_rate": 0.0003441446271070816, + "loss": 0.9759, + "step": 3710 + }, + { + "epoch": 0.384376937383757, + "grad_norm": 1.4455146789550781, + "learning_rate": 0.0003433847226644653, + "loss": 0.9917, + "step": 3720 + }, + { + "epoch": 0.3854102087208101, + "grad_norm": 0.9426379799842834, + "learning_rate": 0.0003426238141760826, + "loss": 1.1549, + "step": 3730 + }, + { + "epoch": 0.3864434800578632, + "grad_norm": 1.3207985162734985, + "learning_rate": 0.00034186190982300323, + "loss": 1.0742, + "step": 3740 + }, + { + "epoch": 0.38747675139491633, + "grad_norm": 0.9599164128303528, + "learning_rate": 0.00034109901779700474, + "loss": 1.0011, + "step": 3750 + }, + { + "epoch": 0.38851002273196944, + "grad_norm": 1.0271940231323242, + "learning_rate": 0.0003403351463004831, + "loss": 0.9508, + "step": 3760 + }, + { + "epoch": 0.38954329406902255, + "grad_norm": 1.680550456047058, + "learning_rate": 0.0003395703035463659, + "loss": 1.134, + "step": 3770 + }, + { + "epoch": 0.39057656540607566, + "grad_norm": 1.4388446807861328, + "learning_rate": 0.00033880449775802306, + "loss": 1.1811, + "step": 3780 + }, + { + "epoch": 0.39160983674312877, + "grad_norm": 1.1560338735580444, + "learning_rate": 0.00033803773716917896, + "loss": 0.8677, + "step": 3790 + }, + { + "epoch": 0.3926431080801819, + "grad_norm": 1.414915680885315, + "learning_rate": 0.00033727003002382346, + "loss": 1.0965, + "step": 3800 + }, + { + "epoch": 0.393676379417235, + "grad_norm": 1.1397249698638916, + "learning_rate": 0.0003365013845761239, + "loss": 1.0057, + "step": 3810 + }, + { + "epoch": 0.3947096507542881, + "grad_norm": 1.470551609992981, + "learning_rate": 0.00033573180909033567, + "loss": 1.1104, + "step": 3820 + }, + { + "epoch": 0.3957429220913412, + "grad_norm": 0.9758129119873047, + "learning_rate": 0.0003349613118407138, + "loss": 1.0244, + "step": 3830 + }, + { + "epoch": 0.3967761934283943, + "grad_norm": 0.9483941793441772, + "learning_rate": 0.0003341899011114238, + "loss": 1.0993, + "step": 3840 + }, + { + "epoch": 0.39780946476544743, + "grad_norm": 1.0213124752044678, + "learning_rate": 0.00033341758519645273, + "loss": 0.9506, + "step": 3850 + }, + { + "epoch": 0.39884273610250054, + "grad_norm": 1.0135273933410645, + "learning_rate": 0.0003326443723995199, + "loss": 0.993, + "step": 3860 + }, + { + "epoch": 0.39987600743955365, + "grad_norm": 0.9621232748031616, + "learning_rate": 0.0003318702710339875, + "loss": 0.9969, + "step": 3870 + }, + { + "epoch": 0.40090927877660676, + "grad_norm": 1.1387019157409668, + "learning_rate": 0.0003310952894227718, + "loss": 1.0203, + "step": 3880 + }, + { + "epoch": 0.40194255011365987, + "grad_norm": 1.406870722770691, + "learning_rate": 0.0003303194358982526, + "loss": 1.0166, + "step": 3890 + }, + { + "epoch": 0.402975821450713, + "grad_norm": 1.339579701423645, + "learning_rate": 0.00032954271880218465, + "loss": 0.9749, + "step": 3900 + }, + { + "epoch": 0.4040090927877661, + "grad_norm": 1.2406803369522095, + "learning_rate": 0.00032876514648560757, + "loss": 0.943, + "step": 3910 + }, + { + "epoch": 0.4050423641248192, + "grad_norm": 1.172455072402954, + "learning_rate": 0.00032798672730875603, + "loss": 0.9874, + "step": 3920 + }, + { + "epoch": 0.4060756354618723, + "grad_norm": 1.2804698944091797, + "learning_rate": 0.0003272074696409697, + "loss": 0.8747, + "step": 3930 + }, + { + "epoch": 0.4071089067989254, + "grad_norm": 1.3023015260696411, + "learning_rate": 0.00032642738186060375, + "loss": 1.0686, + "step": 3940 + }, + { + "epoch": 0.40814217813597853, + "grad_norm": 1.0053542852401733, + "learning_rate": 0.00032564647235493833, + "loss": 1.074, + "step": 3950 + }, + { + "epoch": 0.40917544947303164, + "grad_norm": 1.0846545696258545, + "learning_rate": 0.0003248647495200886, + "loss": 1.0043, + "step": 3960 + }, + { + "epoch": 0.41020872081008475, + "grad_norm": 1.0012749433517456, + "learning_rate": 0.00032408222176091427, + "loss": 1.0302, + "step": 3970 + }, + { + "epoch": 0.41124199214713786, + "grad_norm": 1.074317455291748, + "learning_rate": 0.0003232988974909296, + "loss": 0.9389, + "step": 3980 + }, + { + "epoch": 0.41227526348419097, + "grad_norm": 1.5110340118408203, + "learning_rate": 0.00032251478513221254, + "loss": 0.9772, + "step": 3990 + }, + { + "epoch": 0.4133085348212441, + "grad_norm": 0.9564982652664185, + "learning_rate": 0.00032172989311531427, + "loss": 1.0754, + "step": 4000 + }, + { + "epoch": 0.4143418061582972, + "grad_norm": 1.091377854347229, + "learning_rate": 0.0003209442298791689, + "loss": 0.9343, + "step": 4010 + }, + { + "epoch": 0.4153750774953503, + "grad_norm": 1.3888299465179443, + "learning_rate": 0.00032015780387100227, + "loss": 0.9448, + "step": 4020 + }, + { + "epoch": 0.4164083488324034, + "grad_norm": 0.9059675335884094, + "learning_rate": 0.0003193706235462412, + "loss": 0.9877, + "step": 4030 + }, + { + "epoch": 0.4174416201694565, + "grad_norm": 1.5517827272415161, + "learning_rate": 0.0003185826973684231, + "loss": 1.0814, + "step": 4040 + }, + { + "epoch": 0.4184748915065096, + "grad_norm": 1.1768487691879272, + "learning_rate": 0.00031779403380910427, + "loss": 1.0654, + "step": 4050 + }, + { + "epoch": 0.41950816284356274, + "grad_norm": 1.0996865034103394, + "learning_rate": 0.0003170046413477692, + "loss": 1.0507, + "step": 4060 + }, + { + "epoch": 0.42054143418061585, + "grad_norm": 1.7401593923568726, + "learning_rate": 0.00031621452847173923, + "loss": 1.1611, + "step": 4070 + }, + { + "epoch": 0.42157470551766896, + "grad_norm": 1.4233949184417725, + "learning_rate": 0.00031542370367608176, + "loss": 1.0043, + "step": 4080 + }, + { + "epoch": 0.42260797685472207, + "grad_norm": 1.1471216678619385, + "learning_rate": 0.00031463217546351803, + "loss": 1.0389, + "step": 4090 + }, + { + "epoch": 0.4236412481917752, + "grad_norm": 0.9187898635864258, + "learning_rate": 0.0003138399523443326, + "loss": 0.9048, + "step": 4100 + }, + { + "epoch": 0.4246745195288283, + "grad_norm": 1.0712145566940308, + "learning_rate": 0.0003130470428362811, + "loss": 0.9538, + "step": 4110 + }, + { + "epoch": 0.4257077908658814, + "grad_norm": 1.3978677988052368, + "learning_rate": 0.0003122534554644995, + "loss": 0.9093, + "step": 4120 + }, + { + "epoch": 0.4267410622029345, + "grad_norm": 0.9700504541397095, + "learning_rate": 0.0003114591987614117, + "loss": 0.984, + "step": 4130 + }, + { + "epoch": 0.4277743335399876, + "grad_norm": 1.1521259546279907, + "learning_rate": 0.0003106642812666379, + "loss": 0.9965, + "step": 4140 + }, + { + "epoch": 0.4288076048770407, + "grad_norm": 1.2085431814193726, + "learning_rate": 0.0003098687115269034, + "loss": 0.9018, + "step": 4150 + }, + { + "epoch": 0.42984087621409384, + "grad_norm": 1.1607948541641235, + "learning_rate": 0.0003090724980959457, + "loss": 1.0776, + "step": 4160 + }, + { + "epoch": 0.43087414755114695, + "grad_norm": 1.4290366172790527, + "learning_rate": 0.0003082756495344234, + "loss": 0.9008, + "step": 4170 + }, + { + "epoch": 0.43190741888820006, + "grad_norm": 1.3393192291259766, + "learning_rate": 0.00030747817440982385, + "loss": 0.9175, + "step": 4180 + }, + { + "epoch": 0.43294069022525317, + "grad_norm": 1.4794620275497437, + "learning_rate": 0.00030668008129637104, + "loss": 1.0164, + "step": 4190 + }, + { + "epoch": 0.4339739615623063, + "grad_norm": 0.5630869269371033, + "learning_rate": 0.00030588137877493317, + "loss": 1.0511, + "step": 4200 + }, + { + "epoch": 0.4350072328993594, + "grad_norm": 1.0285252332687378, + "learning_rate": 0.0003050820754329309, + "loss": 0.8834, + "step": 4210 + }, + { + "epoch": 0.4360405042364125, + "grad_norm": 1.4534152746200562, + "learning_rate": 0.00030428217986424444, + "loss": 1.0248, + "step": 4220 + }, + { + "epoch": 0.4370737755734656, + "grad_norm": 1.7549463510513306, + "learning_rate": 0.0003034817006691219, + "loss": 0.9278, + "step": 4230 + }, + { + "epoch": 0.4381070469105187, + "grad_norm": 1.1008073091506958, + "learning_rate": 0.0003026806464540859, + "loss": 1.0172, + "step": 4240 + }, + { + "epoch": 0.4391403182475718, + "grad_norm": 1.042400598526001, + "learning_rate": 0.0003018790258318417, + "loss": 0.9703, + "step": 4250 + }, + { + "epoch": 0.44017358958462494, + "grad_norm": 1.428467035293579, + "learning_rate": 0.00030107684742118466, + "loss": 0.9187, + "step": 4260 + }, + { + "epoch": 0.44120686092167805, + "grad_norm": 0.8269082903862, + "learning_rate": 0.0003002741198469068, + "loss": 0.8501, + "step": 4270 + }, + { + "epoch": 0.44224013225873116, + "grad_norm": 1.0836302042007446, + "learning_rate": 0.00029947085173970506, + "loss": 0.9252, + "step": 4280 + }, + { + "epoch": 0.44327340359578427, + "grad_norm": 0.8747875690460205, + "learning_rate": 0.00029866705173608776, + "loss": 0.8576, + "step": 4290 + }, + { + "epoch": 0.4443066749328374, + "grad_norm": 1.261354923248291, + "learning_rate": 0.0002978627284782821, + "loss": 0.8515, + "step": 4300 + }, + { + "epoch": 0.4453399462698905, + "grad_norm": 1.2424492835998535, + "learning_rate": 0.0002970578906141411, + "loss": 0.9274, + "step": 4310 + }, + { + "epoch": 0.4463732176069436, + "grad_norm": 1.6803146600723267, + "learning_rate": 0.00029625254679705094, + "loss": 1.1236, + "step": 4320 + }, + { + "epoch": 0.4474064889439967, + "grad_norm": 1.284347653388977, + "learning_rate": 0.00029544670568583734, + "loss": 0.9698, + "step": 4330 + }, + { + "epoch": 0.4484397602810498, + "grad_norm": 0.7693723440170288, + "learning_rate": 0.00029464037594467284, + "loss": 0.9518, + "step": 4340 + }, + { + "epoch": 0.4494730316181029, + "grad_norm": 1.0315730571746826, + "learning_rate": 0.00029383356624298383, + "loss": 1.0873, + "step": 4350 + }, + { + "epoch": 0.45050630295515603, + "grad_norm": 0.8411365747451782, + "learning_rate": 0.0002930262852553568, + "loss": 1.0025, + "step": 4360 + }, + { + "epoch": 0.45153957429220914, + "grad_norm": 0.8453785181045532, + "learning_rate": 0.0002922185416614456, + "loss": 0.8811, + "step": 4370 + }, + { + "epoch": 0.45257284562926225, + "grad_norm": 1.2322771549224854, + "learning_rate": 0.0002914103441458775, + "loss": 1.0246, + "step": 4380 + }, + { + "epoch": 0.45360611696631536, + "grad_norm": 1.2167178392410278, + "learning_rate": 0.0002906017013981609, + "loss": 0.9645, + "step": 4390 + }, + { + "epoch": 0.4546393883033685, + "grad_norm": 1.0350911617279053, + "learning_rate": 0.0002897926221125906, + "loss": 0.9242, + "step": 4400 + }, + { + "epoch": 0.4556726596404216, + "grad_norm": 1.1375268697738647, + "learning_rate": 0.0002889831149881551, + "loss": 0.945, + "step": 4410 + }, + { + "epoch": 0.4567059309774747, + "grad_norm": 1.209620475769043, + "learning_rate": 0.0002881731887284429, + "loss": 1.0773, + "step": 4420 + }, + { + "epoch": 0.4577392023145278, + "grad_norm": 1.2932454347610474, + "learning_rate": 0.0002873628520415491, + "loss": 0.8517, + "step": 4430 + }, + { + "epoch": 0.4587724736515809, + "grad_norm": 1.0030730962753296, + "learning_rate": 0.0002865521136399814, + "loss": 0.8911, + "step": 4440 + }, + { + "epoch": 0.459805744988634, + "grad_norm": 0.9559937119483948, + "learning_rate": 0.0002857409822405666, + "loss": 0.9505, + "step": 4450 + }, + { + "epoch": 0.46083901632568713, + "grad_norm": 0.934845507144928, + "learning_rate": 0.00028492946656435704, + "loss": 1.0858, + "step": 4460 + }, + { + "epoch": 0.46187228766274024, + "grad_norm": 1.2203209400177002, + "learning_rate": 0.00028411757533653657, + "loss": 0.9919, + "step": 4470 + }, + { + "epoch": 0.46290555899979335, + "grad_norm": 1.3204402923583984, + "learning_rate": 0.000283305317286327, + "loss": 0.9198, + "step": 4480 + }, + { + "epoch": 0.46393883033684646, + "grad_norm": 0.7426913380622864, + "learning_rate": 0.0002824927011468939, + "loss": 1.0825, + "step": 4490 + }, + { + "epoch": 0.4649721016738996, + "grad_norm": 0.8991280198097229, + "learning_rate": 0.00028167973565525325, + "loss": 0.9528, + "step": 4500 + }, + { + "epoch": 0.4660053730109527, + "grad_norm": 0.697215735912323, + "learning_rate": 0.00028086642955217666, + "loss": 0.8118, + "step": 4510 + }, + { + "epoch": 0.4670386443480058, + "grad_norm": 1.4364171028137207, + "learning_rate": 0.0002800527915820984, + "loss": 0.8903, + "step": 4520 + }, + { + "epoch": 0.4680719156850589, + "grad_norm": 0.9677620530128479, + "learning_rate": 0.00027923883049302067, + "loss": 0.9083, + "step": 4530 + }, + { + "epoch": 0.469105187022112, + "grad_norm": 1.240580439567566, + "learning_rate": 0.00027842455503641965, + "loss": 0.8643, + "step": 4540 + }, + { + "epoch": 0.4701384583591651, + "grad_norm": 1.102919101715088, + "learning_rate": 0.00027760997396715167, + "loss": 1.0802, + "step": 4550 + }, + { + "epoch": 0.47117172969621823, + "grad_norm": 0.921911895275116, + "learning_rate": 0.0002767950960433588, + "loss": 0.8717, + "step": 4560 + }, + { + "epoch": 0.47220500103327134, + "grad_norm": 1.3429372310638428, + "learning_rate": 0.000275979930026375, + "loss": 0.9584, + "step": 4570 + }, + { + "epoch": 0.47323827237032445, + "grad_norm": 1.0819973945617676, + "learning_rate": 0.0002751644846806315, + "loss": 0.7652, + "step": 4580 + }, + { + "epoch": 0.47427154370737756, + "grad_norm": 0.9730139970779419, + "learning_rate": 0.00027434876877356287, + "loss": 0.8873, + "step": 4590 + }, + { + "epoch": 0.4753048150444307, + "grad_norm": 0.9837844371795654, + "learning_rate": 0.00027353279107551276, + "loss": 0.9581, + "step": 4600 + }, + { + "epoch": 0.4763380863814838, + "grad_norm": 1.1134988069534302, + "learning_rate": 0.0002727165603596394, + "loss": 0.9409, + "step": 4610 + }, + { + "epoch": 0.4773713577185369, + "grad_norm": 0.7778913378715515, + "learning_rate": 0.00027190008540182135, + "loss": 0.9099, + "step": 4620 + }, + { + "epoch": 0.47840462905559, + "grad_norm": 1.166568636894226, + "learning_rate": 0.00027108337498056356, + "loss": 0.8565, + "step": 4630 + }, + { + "epoch": 0.4794379003926431, + "grad_norm": 0.7446126341819763, + "learning_rate": 0.0002702664378769021, + "loss": 0.8251, + "step": 4640 + }, + { + "epoch": 0.4804711717296962, + "grad_norm": 1.5371274948120117, + "learning_rate": 0.0002694492828743106, + "loss": 0.9475, + "step": 4650 + }, + { + "epoch": 0.48150444306674933, + "grad_norm": 0.7517269253730774, + "learning_rate": 0.0002686319187586054, + "loss": 0.8798, + "step": 4660 + }, + { + "epoch": 0.48253771440380244, + "grad_norm": 1.4352338314056396, + "learning_rate": 0.00026781435431785116, + "loss": 1.0462, + "step": 4670 + }, + { + "epoch": 0.48357098574085555, + "grad_norm": 1.1265496015548706, + "learning_rate": 0.0002669965983422663, + "loss": 0.9107, + "step": 4680 + }, + { + "epoch": 0.48460425707790866, + "grad_norm": 0.8634279370307922, + "learning_rate": 0.00026617865962412865, + "loss": 0.8822, + "step": 4690 + }, + { + "epoch": 0.48563752841496177, + "grad_norm": 1.0112096071243286, + "learning_rate": 0.0002653605469576809, + "loss": 0.8842, + "step": 4700 + }, + { + "epoch": 0.4866707997520149, + "grad_norm": 0.8251860737800598, + "learning_rate": 0.0002645422691390358, + "loss": 0.908, + "step": 4710 + }, + { + "epoch": 0.487704071089068, + "grad_norm": 0.7483717203140259, + "learning_rate": 0.0002637238349660819, + "loss": 0.8912, + "step": 4720 + }, + { + "epoch": 0.4887373424261211, + "grad_norm": 0.8353511691093445, + "learning_rate": 0.0002629052532383888, + "loss": 0.7702, + "step": 4730 + }, + { + "epoch": 0.4897706137631742, + "grad_norm": 0.9910038113594055, + "learning_rate": 0.00026208653275711265, + "loss": 0.8797, + "step": 4740 + }, + { + "epoch": 0.4908038851002273, + "grad_norm": 1.1162399053573608, + "learning_rate": 0.00026126768232490115, + "loss": 0.9161, + "step": 4750 + }, + { + "epoch": 0.49183715643728043, + "grad_norm": 0.8416900038719177, + "learning_rate": 0.00026044871074579955, + "loss": 1.0406, + "step": 4760 + }, + { + "epoch": 0.49287042777433354, + "grad_norm": 1.2481132745742798, + "learning_rate": 0.0002596296268251556, + "loss": 1.0545, + "step": 4770 + }, + { + "epoch": 0.49390369911138665, + "grad_norm": 1.115768551826477, + "learning_rate": 0.0002588104393695245, + "loss": 0.9128, + "step": 4780 + }, + { + "epoch": 0.49493697044843976, + "grad_norm": 0.7396090030670166, + "learning_rate": 0.0002579911571865752, + "loss": 1.0822, + "step": 4790 + }, + { + "epoch": 0.49597024178549287, + "grad_norm": 0.9519025087356567, + "learning_rate": 0.0002571717890849946, + "loss": 0.8659, + "step": 4800 + }, + { + "epoch": 0.497003513122546, + "grad_norm": 0.7343184947967529, + "learning_rate": 0.0002563523438743939, + "loss": 0.8773, + "step": 4810 + }, + { + "epoch": 0.4980367844595991, + "grad_norm": 1.4175959825515747, + "learning_rate": 0.0002555328303652129, + "loss": 0.7956, + "step": 4820 + }, + { + "epoch": 0.4990700557966522, + "grad_norm": 1.0520236492156982, + "learning_rate": 0.000254713257368626, + "loss": 0.9604, + "step": 4830 + }, + { + "epoch": 0.5001033271337053, + "grad_norm": 0.7733720541000366, + "learning_rate": 0.0002538936336964471, + "loss": 0.7715, + "step": 4840 + }, + { + "epoch": 0.5011365984707584, + "grad_norm": 1.4437015056610107, + "learning_rate": 0.0002530739681610349, + "loss": 0.9329, + "step": 4850 + }, + { + "epoch": 0.5021698698078115, + "grad_norm": 0.892091691493988, + "learning_rate": 0.00025225426957519825, + "loss": 0.8434, + "step": 4860 + }, + { + "epoch": 0.5032031411448646, + "grad_norm": 0.9304814338684082, + "learning_rate": 0.0002514345467521014, + "loss": 0.8912, + "step": 4870 + }, + { + "epoch": 0.5042364124819178, + "grad_norm": 1.063408374786377, + "learning_rate": 0.00025061480850516914, + "loss": 0.9901, + "step": 4880 + }, + { + "epoch": 0.5052696838189709, + "grad_norm": 1.0481808185577393, + "learning_rate": 0.00024979506364799207, + "loss": 0.89, + "step": 4890 + }, + { + "epoch": 0.506302955156024, + "grad_norm": 1.2767094373703003, + "learning_rate": 0.000248975320994232, + "loss": 1.1239, + "step": 4900 + }, + { + "epoch": 0.5073362264930771, + "grad_norm": 1.0354382991790771, + "learning_rate": 0.00024815558935752677, + "loss": 0.8866, + "step": 4910 + }, + { + "epoch": 0.5083694978301302, + "grad_norm": 0.7483660578727722, + "learning_rate": 0.0002473358775513959, + "loss": 0.8817, + "step": 4920 + }, + { + "epoch": 0.5094027691671833, + "grad_norm": 0.6301681399345398, + "learning_rate": 0.000246516194389146, + "loss": 0.8991, + "step": 4930 + }, + { + "epoch": 0.5104360405042364, + "grad_norm": 1.0671608448028564, + "learning_rate": 0.0002456965486837752, + "loss": 0.7891, + "step": 4940 + }, + { + "epoch": 0.5114693118412895, + "grad_norm": 1.3131459951400757, + "learning_rate": 0.00024487694924787935, + "loss": 0.9388, + "step": 4950 + }, + { + "epoch": 0.5125025831783426, + "grad_norm": 1.2082393169403076, + "learning_rate": 0.00024405740489355634, + "loss": 0.8558, + "step": 4960 + }, + { + "epoch": 0.5135358545153957, + "grad_norm": 0.915107786655426, + "learning_rate": 0.0002432379244323124, + "loss": 0.8703, + "step": 4970 + }, + { + "epoch": 0.5145691258524488, + "grad_norm": 1.2518354654312134, + "learning_rate": 0.00024241851667496645, + "loss": 0.9959, + "step": 4980 + }, + { + "epoch": 0.515602397189502, + "grad_norm": 1.0495281219482422, + "learning_rate": 0.00024159919043155556, + "loss": 1.0497, + "step": 4990 + }, + { + "epoch": 0.5166356685265551, + "grad_norm": 1.469854712486267, + "learning_rate": 0.0002407799545112409, + "loss": 0.7946, + "step": 5000 + }, + { + "epoch": 0.5176689398636082, + "grad_norm": 0.7165640592575073, + "learning_rate": 0.0002399608177222122, + "loss": 0.7768, + "step": 5010 + }, + { + "epoch": 0.5187022112006613, + "grad_norm": 0.7148188352584839, + "learning_rate": 0.00023914178887159327, + "loss": 0.8901, + "step": 5020 + }, + { + "epoch": 0.5197354825377144, + "grad_norm": 1.2880741357803345, + "learning_rate": 0.0002383228767653474, + "loss": 1.0241, + "step": 5030 + }, + { + "epoch": 0.5207687538747675, + "grad_norm": 1.3181527853012085, + "learning_rate": 0.0002375040902081831, + "loss": 0.9248, + "step": 5040 + }, + { + "epoch": 0.5218020252118206, + "grad_norm": 1.5515258312225342, + "learning_rate": 0.00023668543800345852, + "loss": 0.8286, + "step": 5050 + }, + { + "epoch": 0.5228352965488737, + "grad_norm": 1.0438510179519653, + "learning_rate": 0.0002358669289530875, + "loss": 0.7546, + "step": 5060 + }, + { + "epoch": 0.5238685678859268, + "grad_norm": 1.179828405380249, + "learning_rate": 0.00023504857185744453, + "loss": 0.8099, + "step": 5070 + }, + { + "epoch": 0.52490183922298, + "grad_norm": 1.0748212337493896, + "learning_rate": 0.00023423037551527085, + "loss": 0.8692, + "step": 5080 + }, + { + "epoch": 0.5259351105600331, + "grad_norm": 0.8671577572822571, + "learning_rate": 0.00023341234872357872, + "loss": 0.8423, + "step": 5090 + }, + { + "epoch": 0.5269683818970862, + "grad_norm": 1.147182583808899, + "learning_rate": 0.00023259450027755764, + "loss": 0.8823, + "step": 5100 + }, + { + "epoch": 0.5280016532341393, + "grad_norm": 1.0983150005340576, + "learning_rate": 0.0002317768389704799, + "loss": 0.879, + "step": 5110 + }, + { + "epoch": 0.5290349245711924, + "grad_norm": 1.5440839529037476, + "learning_rate": 0.00023095937359360525, + "loss": 0.9303, + "step": 5120 + }, + { + "epoch": 0.5300681959082455, + "grad_norm": 1.0175645351409912, + "learning_rate": 0.00023014211293608718, + "loss": 0.8823, + "step": 5130 + }, + { + "epoch": 0.5311014672452986, + "grad_norm": 1.1853641271591187, + "learning_rate": 0.00022932506578487782, + "loss": 0.9467, + "step": 5140 + }, + { + "epoch": 0.5321347385823517, + "grad_norm": 0.9132509827613831, + "learning_rate": 0.00022850824092463413, + "loss": 0.7992, + "step": 5150 + }, + { + "epoch": 0.5331680099194048, + "grad_norm": 0.8958359360694885, + "learning_rate": 0.0002276916471376227, + "loss": 0.9003, + "step": 5160 + }, + { + "epoch": 0.5342012812564579, + "grad_norm": 0.9055427312850952, + "learning_rate": 0.00022687529320362587, + "loss": 0.8922, + "step": 5170 + }, + { + "epoch": 0.535234552593511, + "grad_norm": 1.2413357496261597, + "learning_rate": 0.00022605918789984707, + "loss": 0.8021, + "step": 5180 + }, + { + "epoch": 0.5362678239305642, + "grad_norm": 1.096956729888916, + "learning_rate": 0.00022524334000081664, + "loss": 0.7335, + "step": 5190 + }, + { + "epoch": 0.5373010952676173, + "grad_norm": 1.0181100368499756, + "learning_rate": 0.00022442775827829722, + "loss": 0.9217, + "step": 5200 + }, + { + "epoch": 0.5383343666046704, + "grad_norm": 0.9444778561592102, + "learning_rate": 0.0002236124515011897, + "loss": 0.8444, + "step": 5210 + }, + { + "epoch": 0.5393676379417235, + "grad_norm": 0.9442963004112244, + "learning_rate": 0.00022279742843543887, + "loss": 0.7901, + "step": 5220 + }, + { + "epoch": 0.5404009092787766, + "grad_norm": 1.3280583620071411, + "learning_rate": 0.00022198269784393904, + "loss": 0.9, + "step": 5230 + }, + { + "epoch": 0.5414341806158297, + "grad_norm": 0.7821478843688965, + "learning_rate": 0.00022116826848643994, + "loss": 0.8782, + "step": 5240 + }, + { + "epoch": 0.5424674519528828, + "grad_norm": 0.942866861820221, + "learning_rate": 0.00022035414911945252, + "loss": 0.8268, + "step": 5250 + }, + { + "epoch": 0.5435007232899359, + "grad_norm": 1.0075554847717285, + "learning_rate": 0.0002195403484961549, + "loss": 0.7923, + "step": 5260 + }, + { + "epoch": 0.544533994626989, + "grad_norm": 1.0974137783050537, + "learning_rate": 0.00021872687536629806, + "loss": 0.8108, + "step": 5270 + }, + { + "epoch": 0.5455672659640421, + "grad_norm": 0.8785361647605896, + "learning_rate": 0.00021791373847611186, + "loss": 0.8236, + "step": 5280 + }, + { + "epoch": 0.5466005373010953, + "grad_norm": 1.300614356994629, + "learning_rate": 0.00021710094656821098, + "loss": 0.9875, + "step": 5290 + }, + { + "epoch": 0.5476338086381484, + "grad_norm": 0.9605757594108582, + "learning_rate": 0.0002162885083815011, + "loss": 0.8824, + "step": 5300 + }, + { + "epoch": 0.5486670799752015, + "grad_norm": 0.8986537456512451, + "learning_rate": 0.0002154764326510847, + "loss": 0.9041, + "step": 5310 + }, + { + "epoch": 0.5497003513122546, + "grad_norm": 0.8150350451469421, + "learning_rate": 0.0002146647281081671, + "loss": 0.8553, + "step": 5320 + }, + { + "epoch": 0.5507336226493077, + "grad_norm": 1.396665334701538, + "learning_rate": 0.00021385340347996297, + "loss": 0.9266, + "step": 5330 + }, + { + "epoch": 0.5517668939863608, + "grad_norm": 1.1270840167999268, + "learning_rate": 0.00021304246748960208, + "loss": 0.8426, + "step": 5340 + }, + { + "epoch": 0.5528001653234139, + "grad_norm": 1.4210362434387207, + "learning_rate": 0.00021223192885603571, + "loss": 0.9085, + "step": 5350 + }, + { + "epoch": 0.553833436660467, + "grad_norm": 0.9129871726036072, + "learning_rate": 0.00021142179629394283, + "loss": 0.8978, + "step": 5360 + }, + { + "epoch": 0.5548667079975201, + "grad_norm": 1.0395097732543945, + "learning_rate": 0.00021061207851363661, + "loss": 0.9224, + "step": 5370 + }, + { + "epoch": 0.5558999793345732, + "grad_norm": 0.5978155136108398, + "learning_rate": 0.00020980278422097045, + "loss": 0.789, + "step": 5380 + }, + { + "epoch": 0.5569332506716264, + "grad_norm": 0.8347190022468567, + "learning_rate": 0.0002089939221172446, + "loss": 0.8651, + "step": 5390 + }, + { + "epoch": 0.5579665220086795, + "grad_norm": 1.0471965074539185, + "learning_rate": 0.00020818550089911234, + "loss": 0.9259, + "step": 5400 + }, + { + "epoch": 0.5589997933457326, + "grad_norm": 0.8594735264778137, + "learning_rate": 0.00020737752925848707, + "loss": 0.7926, + "step": 5410 + }, + { + "epoch": 0.5600330646827857, + "grad_norm": 0.9445158243179321, + "learning_rate": 0.00020657001588244816, + "loss": 0.9561, + "step": 5420 + }, + { + "epoch": 0.5610663360198388, + "grad_norm": 0.7682790160179138, + "learning_rate": 0.00020576296945314787, + "loss": 0.8636, + "step": 5430 + }, + { + "epoch": 0.5620996073568919, + "grad_norm": 0.8511012196540833, + "learning_rate": 0.00020495639864771814, + "loss": 0.877, + "step": 5440 + }, + { + "epoch": 0.563132878693945, + "grad_norm": 1.3375914096832275, + "learning_rate": 0.00020415031213817704, + "loss": 0.8865, + "step": 5450 + }, + { + "epoch": 0.5641661500309981, + "grad_norm": 1.4135067462921143, + "learning_rate": 0.0002033447185913356, + "loss": 0.8816, + "step": 5460 + }, + { + "epoch": 0.5651994213680512, + "grad_norm": 1.0292867422103882, + "learning_rate": 0.0002025396266687045, + "loss": 0.7509, + "step": 5470 + }, + { + "epoch": 0.5662326927051043, + "grad_norm": 0.8321288228034973, + "learning_rate": 0.00020173504502640164, + "loss": 0.8376, + "step": 5480 + }, + { + "epoch": 0.5672659640421575, + "grad_norm": 0.7995291948318481, + "learning_rate": 0.000200930982315058, + "loss": 0.822, + "step": 5490 + }, + { + "epoch": 0.5682992353792106, + "grad_norm": 0.7237703800201416, + "learning_rate": 0.0002001274471797252, + "loss": 0.9819, + "step": 5500 + }, + { + "epoch": 0.5693325067162637, + "grad_norm": 1.2666767835617065, + "learning_rate": 0.0001993244482597829, + "loss": 0.7477, + "step": 5510 + }, + { + "epoch": 0.5703657780533168, + "grad_norm": 1.1628350019454956, + "learning_rate": 0.00019852199418884527, + "loss": 0.7239, + "step": 5520 + }, + { + "epoch": 0.5713990493903699, + "grad_norm": 1.1754180192947388, + "learning_rate": 0.00019772009359466828, + "loss": 0.8326, + "step": 5530 + }, + { + "epoch": 0.572432320727423, + "grad_norm": 0.9641863703727722, + "learning_rate": 0.00019691875509905718, + "loss": 0.779, + "step": 5540 + }, + { + "epoch": 0.5734655920644761, + "grad_norm": 1.4042216539382935, + "learning_rate": 0.00019611798731777397, + "loss": 0.8895, + "step": 5550 + }, + { + "epoch": 0.5744988634015292, + "grad_norm": 0.8444200158119202, + "learning_rate": 0.00019531779886044424, + "loss": 0.8636, + "step": 5560 + }, + { + "epoch": 0.5755321347385823, + "grad_norm": 0.8476575016975403, + "learning_rate": 0.0001945181983304647, + "loss": 0.802, + "step": 5570 + }, + { + "epoch": 0.5765654060756354, + "grad_norm": 1.086524486541748, + "learning_rate": 0.00019371919432491108, + "loss": 0.8356, + "step": 5580 + }, + { + "epoch": 0.5775986774126886, + "grad_norm": 0.8224913477897644, + "learning_rate": 0.00019292079543444557, + "loss": 0.8772, + "step": 5590 + }, + { + "epoch": 0.5786319487497417, + "grad_norm": 0.8469100594520569, + "learning_rate": 0.00019212301024322396, + "loss": 0.883, + "step": 5600 + }, + { + "epoch": 0.5796652200867948, + "grad_norm": 0.6162629127502441, + "learning_rate": 0.00019132584732880383, + "loss": 0.7449, + "step": 5610 + }, + { + "epoch": 0.5806984914238479, + "grad_norm": 1.1471900939941406, + "learning_rate": 0.00019052931526205267, + "loss": 0.8848, + "step": 5620 + }, + { + "epoch": 0.581731762760901, + "grad_norm": 1.4094953536987305, + "learning_rate": 0.00018973342260705458, + "loss": 0.7942, + "step": 5630 + }, + { + "epoch": 0.5827650340979541, + "grad_norm": 0.8406280279159546, + "learning_rate": 0.00018893817792101935, + "loss": 0.8205, + "step": 5640 + }, + { + "epoch": 0.5837983054350072, + "grad_norm": 1.5820422172546387, + "learning_rate": 0.00018814358975418984, + "loss": 0.733, + "step": 5650 + }, + { + "epoch": 0.5848315767720603, + "grad_norm": 1.0317678451538086, + "learning_rate": 0.00018734966664975023, + "loss": 0.8889, + "step": 5660 + }, + { + "epoch": 0.5858648481091134, + "grad_norm": 1.1359983682632446, + "learning_rate": 0.00018655641714373413, + "loss": 0.897, + "step": 5670 + }, + { + "epoch": 0.5868981194461665, + "grad_norm": 1.0122281312942505, + "learning_rate": 0.00018576384976493282, + "loss": 0.7574, + "step": 5680 + }, + { + "epoch": 0.5879313907832197, + "grad_norm": 0.9288797378540039, + "learning_rate": 0.00018497197303480348, + "loss": 0.7553, + "step": 5690 + }, + { + "epoch": 0.5889646621202728, + "grad_norm": 0.8775319457054138, + "learning_rate": 0.00018418079546737783, + "loss": 0.74, + "step": 5700 + }, + { + "epoch": 0.5899979334573259, + "grad_norm": 1.075302243232727, + "learning_rate": 0.0001833903255691702, + "loss": 0.8761, + "step": 5710 + }, + { + "epoch": 0.591031204794379, + "grad_norm": 1.0737923383712769, + "learning_rate": 0.00018260057183908625, + "loss": 0.8817, + "step": 5720 + }, + { + "epoch": 0.5920644761314321, + "grad_norm": 1.1469508409500122, + "learning_rate": 0.00018181154276833178, + "loss": 0.7806, + "step": 5730 + }, + { + "epoch": 0.5930977474684852, + "grad_norm": 0.8804445266723633, + "learning_rate": 0.00018102324684032115, + "loss": 0.7683, + "step": 5740 + }, + { + "epoch": 0.5941310188055383, + "grad_norm": 0.9593983888626099, + "learning_rate": 0.00018023569253058613, + "loss": 0.8427, + "step": 5750 + }, + { + "epoch": 0.5951642901425914, + "grad_norm": 0.7528694272041321, + "learning_rate": 0.00017944888830668488, + "loss": 0.8697, + "step": 5760 + }, + { + "epoch": 0.5961975614796445, + "grad_norm": 1.0688844919204712, + "learning_rate": 0.00017866284262811093, + "loss": 0.779, + "step": 5770 + }, + { + "epoch": 0.5972308328166976, + "grad_norm": 1.181410312652588, + "learning_rate": 0.00017787756394620202, + "loss": 0.8387, + "step": 5780 + }, + { + "epoch": 0.5982641041537508, + "grad_norm": 0.7557913064956665, + "learning_rate": 0.0001770930607040494, + "loss": 0.7897, + "step": 5790 + }, + { + "epoch": 0.5992973754908039, + "grad_norm": 0.9230046272277832, + "learning_rate": 0.00017630934133640704, + "loss": 0.7056, + "step": 5800 + }, + { + "epoch": 0.600330646827857, + "grad_norm": 0.839525043964386, + "learning_rate": 0.00017552641426960092, + "loss": 0.8133, + "step": 5810 + }, + { + "epoch": 0.6013639181649101, + "grad_norm": 1.102609395980835, + "learning_rate": 0.0001747442879214384, + "loss": 0.884, + "step": 5820 + }, + { + "epoch": 0.6023971895019632, + "grad_norm": 0.8469570875167847, + "learning_rate": 0.0001739629707011177, + "loss": 0.7398, + "step": 5830 + }, + { + "epoch": 0.6034304608390163, + "grad_norm": 1.2042063474655151, + "learning_rate": 0.0001731824710091377, + "loss": 0.8074, + "step": 5840 + }, + { + "epoch": 0.6044637321760694, + "grad_norm": 2.6056320667266846, + "learning_rate": 0.00017240279723720732, + "loss": 0.8092, + "step": 5850 + }, + { + "epoch": 0.6054970035131225, + "grad_norm": 0.788226306438446, + "learning_rate": 0.00017162395776815532, + "loss": 0.8346, + "step": 5860 + }, + { + "epoch": 0.6065302748501756, + "grad_norm": 0.8313378095626831, + "learning_rate": 0.00017084596097584044, + "loss": 0.7632, + "step": 5870 + }, + { + "epoch": 0.6075635461872287, + "grad_norm": 0.9668551087379456, + "learning_rate": 0.00017006881522506123, + "loss": 0.7813, + "step": 5880 + }, + { + "epoch": 0.6085968175242819, + "grad_norm": 1.2980097532272339, + "learning_rate": 0.00016929252887146597, + "loss": 0.633, + "step": 5890 + }, + { + "epoch": 0.609630088861335, + "grad_norm": 1.296339511871338, + "learning_rate": 0.00016851711026146294, + "loss": 0.8067, + "step": 5900 + }, + { + "epoch": 0.6106633601983881, + "grad_norm": 0.8966313600540161, + "learning_rate": 0.00016774256773213075, + "loss": 0.8509, + "step": 5910 + }, + { + "epoch": 0.6116966315354412, + "grad_norm": 1.408347249031067, + "learning_rate": 0.00016696890961112867, + "loss": 0.6981, + "step": 5920 + }, + { + "epoch": 0.6127299028724943, + "grad_norm": 1.421396017074585, + "learning_rate": 0.00016619614421660702, + "loss": 0.7626, + "step": 5930 + }, + { + "epoch": 0.6137631742095474, + "grad_norm": 1.4539517164230347, + "learning_rate": 0.0001654242798571177, + "loss": 0.7558, + "step": 5940 + }, + { + "epoch": 0.6147964455466005, + "grad_norm": 1.0731654167175293, + "learning_rate": 0.0001646533248315252, + "loss": 0.7626, + "step": 5950 + }, + { + "epoch": 0.6158297168836536, + "grad_norm": 1.100475788116455, + "learning_rate": 0.00016388328742891677, + "loss": 0.7243, + "step": 5960 + }, + { + "epoch": 0.6168629882207067, + "grad_norm": 1.0175180435180664, + "learning_rate": 0.00016311417592851396, + "loss": 0.8071, + "step": 5970 + }, + { + "epoch": 0.6178962595577598, + "grad_norm": 0.9055405259132385, + "learning_rate": 0.00016234599859958287, + "loss": 0.7203, + "step": 5980 + }, + { + "epoch": 0.618929530894813, + "grad_norm": 1.0334464311599731, + "learning_rate": 0.00016157876370134617, + "loss": 0.706, + "step": 5990 + }, + { + "epoch": 0.6199628022318661, + "grad_norm": 1.1770230531692505, + "learning_rate": 0.00016081247948289353, + "loss": 0.7802, + "step": 6000 + }, + { + "epoch": 0.6209960735689192, + "grad_norm": 0.8190944790840149, + "learning_rate": 0.00016004715418309304, + "loss": 0.7762, + "step": 6010 + }, + { + "epoch": 0.6220293449059723, + "grad_norm": 1.0050560235977173, + "learning_rate": 0.0001592827960305029, + "loss": 0.7886, + "step": 6020 + }, + { + "epoch": 0.6230626162430254, + "grad_norm": 1.1464440822601318, + "learning_rate": 0.000158519413243283, + "loss": 0.8016, + "step": 6030 + }, + { + "epoch": 0.6240958875800785, + "grad_norm": 0.8896029591560364, + "learning_rate": 0.00015775701402910626, + "loss": 0.802, + "step": 6040 + }, + { + "epoch": 0.6251291589171316, + "grad_norm": 0.7307144403457642, + "learning_rate": 0.00015699560658507012, + "loss": 0.7755, + "step": 6050 + }, + { + "epoch": 0.6261624302541847, + "grad_norm": 0.767782986164093, + "learning_rate": 0.00015623519909760953, + "loss": 0.697, + "step": 6060 + }, + { + "epoch": 0.6271957015912378, + "grad_norm": 0.8861032724380493, + "learning_rate": 0.00015547579974240766, + "loss": 0.8005, + "step": 6070 + }, + { + "epoch": 0.6282289729282909, + "grad_norm": 0.9863024950027466, + "learning_rate": 0.00015471741668430863, + "loss": 0.8324, + "step": 6080 + }, + { + "epoch": 0.629262244265344, + "grad_norm": 0.8046024441719055, + "learning_rate": 0.0001539600580772296, + "loss": 0.8615, + "step": 6090 + }, + { + "epoch": 0.6302955156023972, + "grad_norm": 1.283008098602295, + "learning_rate": 0.0001532037320640734, + "loss": 0.8317, + "step": 6100 + }, + { + "epoch": 0.6313287869394503, + "grad_norm": 0.9851896166801453, + "learning_rate": 0.00015244844677664036, + "loss": 0.7516, + "step": 6110 + }, + { + "epoch": 0.6323620582765034, + "grad_norm": 1.4205191135406494, + "learning_rate": 0.00015169421033554137, + "loss": 0.7692, + "step": 6120 + }, + { + "epoch": 0.6333953296135565, + "grad_norm": 0.9991669058799744, + "learning_rate": 0.0001509410308501104, + "loss": 0.7453, + "step": 6130 + }, + { + "epoch": 0.6344286009506096, + "grad_norm": 0.723731279373169, + "learning_rate": 0.00015018891641831744, + "loss": 0.8026, + "step": 6140 + }, + { + "epoch": 0.6354618722876627, + "grad_norm": 0.5217877626419067, + "learning_rate": 0.00014943787512668118, + "loss": 0.7759, + "step": 6150 + }, + { + "epoch": 0.6364951436247158, + "grad_norm": 0.6816266179084778, + "learning_rate": 0.0001486879150501822, + "loss": 0.7659, + "step": 6160 + }, + { + "epoch": 0.6375284149617689, + "grad_norm": 0.5098093152046204, + "learning_rate": 0.00014793904425217648, + "loss": 0.8352, + "step": 6170 + }, + { + "epoch": 0.638561686298822, + "grad_norm": 1.0227922201156616, + "learning_rate": 0.00014719127078430795, + "loss": 0.8162, + "step": 6180 + }, + { + "epoch": 0.6395949576358751, + "grad_norm": 1.5966726541519165, + "learning_rate": 0.00014644460268642263, + "loss": 0.7707, + "step": 6190 + }, + { + "epoch": 0.6406282289729283, + "grad_norm": 0.6314913630485535, + "learning_rate": 0.00014569904798648176, + "loss": 0.7286, + "step": 6200 + }, + { + "epoch": 0.6416615003099814, + "grad_norm": 1.072977900505066, + "learning_rate": 0.00014495461470047594, + "loss": 0.7554, + "step": 6210 + }, + { + "epoch": 0.6426947716470345, + "grad_norm": 1.458894968032837, + "learning_rate": 0.0001442113108323383, + "loss": 0.8975, + "step": 6220 + }, + { + "epoch": 0.6437280429840876, + "grad_norm": 0.7281268239021301, + "learning_rate": 0.00014346914437385905, + "loss": 0.6558, + "step": 6230 + }, + { + "epoch": 0.6447613143211407, + "grad_norm": 0.9581575393676758, + "learning_rate": 0.00014272812330459893, + "loss": 0.7082, + "step": 6240 + }, + { + "epoch": 0.6457945856581938, + "grad_norm": 1.271925687789917, + "learning_rate": 0.00014198825559180432, + "loss": 0.8919, + "step": 6250 + }, + { + "epoch": 0.6468278569952469, + "grad_norm": 0.9335398077964783, + "learning_rate": 0.00014124954919032064, + "loss": 0.7847, + "step": 6260 + }, + { + "epoch": 0.6478611283323, + "grad_norm": 0.7705923318862915, + "learning_rate": 0.00014051201204250726, + "loss": 0.8197, + "step": 6270 + }, + { + "epoch": 0.6488943996693531, + "grad_norm": 1.433953881263733, + "learning_rate": 0.00013977565207815238, + "loss": 0.8028, + "step": 6280 + }, + { + "epoch": 0.6499276710064062, + "grad_norm": 1.4676005840301514, + "learning_rate": 0.0001390404772143872, + "loss": 0.6803, + "step": 6290 + }, + { + "epoch": 0.6509609423434594, + "grad_norm": 0.8155161738395691, + "learning_rate": 0.00013830649535560093, + "loss": 0.8868, + "step": 6300 + }, + { + "epoch": 0.6519942136805125, + "grad_norm": 0.799152672290802, + "learning_rate": 0.00013757371439335604, + "loss": 0.716, + "step": 6310 + }, + { + "epoch": 0.6530274850175656, + "grad_norm": 1.120835304260254, + "learning_rate": 0.0001368421422063036, + "loss": 0.7953, + "step": 6320 + }, + { + "epoch": 0.6540607563546187, + "grad_norm": 0.7950730919837952, + "learning_rate": 0.00013611178666009794, + "loss": 0.7389, + "step": 6330 + }, + { + "epoch": 0.6550940276916718, + "grad_norm": 0.9817484617233276, + "learning_rate": 0.0001353826556073125, + "loss": 0.7451, + "step": 6340 + }, + { + "epoch": 0.6561272990287249, + "grad_norm": 0.7330102324485779, + "learning_rate": 0.00013465475688735526, + "loss": 0.7386, + "step": 6350 + }, + { + "epoch": 0.657160570365778, + "grad_norm": 0.7877906560897827, + "learning_rate": 0.00013392809832638486, + "loss": 0.7596, + "step": 6360 + }, + { + "epoch": 0.6581938417028311, + "grad_norm": 0.6060128211975098, + "learning_rate": 0.00013320268773722553, + "loss": 0.6482, + "step": 6370 + }, + { + "epoch": 0.6592271130398842, + "grad_norm": 1.8573722839355469, + "learning_rate": 0.00013247853291928395, + "loss": 0.7416, + "step": 6380 + }, + { + "epoch": 0.6602603843769373, + "grad_norm": 0.8654054403305054, + "learning_rate": 0.00013175564165846532, + "loss": 0.7027, + "step": 6390 + }, + { + "epoch": 0.6612936557139905, + "grad_norm": 0.7953044772148132, + "learning_rate": 0.00013103402172708918, + "loss": 0.7331, + "step": 6400 + }, + { + "epoch": 0.6623269270510436, + "grad_norm": 0.8240858316421509, + "learning_rate": 0.0001303136808838061, + "loss": 0.7007, + "step": 6410 + }, + { + "epoch": 0.6633601983880967, + "grad_norm": 0.8028026819229126, + "learning_rate": 0.00012959462687351435, + "loss": 0.7626, + "step": 6420 + }, + { + "epoch": 0.6643934697251498, + "grad_norm": 1.4915801286697388, + "learning_rate": 0.0001288768674272765, + "loss": 0.8101, + "step": 6430 + }, + { + "epoch": 0.6654267410622029, + "grad_norm": 0.8312230110168457, + "learning_rate": 0.0001281604102622364, + "loss": 0.7186, + "step": 6440 + }, + { + "epoch": 0.666460012399256, + "grad_norm": 0.8355895280838013, + "learning_rate": 0.000127445263081536, + "loss": 0.756, + "step": 6450 + }, + { + "epoch": 0.6674932837363091, + "grad_norm": 0.7141729593276978, + "learning_rate": 0.0001267314335742327, + "loss": 0.707, + "step": 6460 + }, + { + "epoch": 0.6685265550733622, + "grad_norm": 1.2061463594436646, + "learning_rate": 0.000126018929415217, + "loss": 0.6433, + "step": 6470 + }, + { + "epoch": 0.6695598264104153, + "grad_norm": 0.9426934123039246, + "learning_rate": 0.00012530775826512916, + "loss": 0.7131, + "step": 6480 + }, + { + "epoch": 0.6705930977474684, + "grad_norm": 0.5171589255332947, + "learning_rate": 0.00012459792777027754, + "loss": 0.6755, + "step": 6490 + }, + { + "epoch": 0.6716263690845216, + "grad_norm": 0.813228189945221, + "learning_rate": 0.00012388944556255614, + "loss": 0.7517, + "step": 6500 + }, + { + "epoch": 0.6726596404215747, + "grad_norm": 1.0219289064407349, + "learning_rate": 0.0001231823192593625, + "loss": 0.647, + "step": 6510 + }, + { + "epoch": 0.6736929117586278, + "grad_norm": 1.7192949056625366, + "learning_rate": 0.00012247655646351597, + "loss": 0.7193, + "step": 6520 + }, + { + "epoch": 0.6747261830956809, + "grad_norm": 0.8978095054626465, + "learning_rate": 0.00012177216476317565, + "loss": 0.7004, + "step": 6530 + }, + { + "epoch": 0.675759454432734, + "grad_norm": 1.2175655364990234, + "learning_rate": 0.00012106915173175942, + "loss": 0.7798, + "step": 6540 + }, + { + "epoch": 0.6767927257697871, + "grad_norm": 0.7553932070732117, + "learning_rate": 0.00012036752492786171, + "loss": 0.7077, + "step": 6550 + }, + { + "epoch": 0.6778259971068402, + "grad_norm": 1.2669484615325928, + "learning_rate": 0.0001196672918951728, + "loss": 0.6207, + "step": 6560 + }, + { + "epoch": 0.6788592684438933, + "grad_norm": 1.0848954916000366, + "learning_rate": 0.00011896846016239752, + "loss": 0.7216, + "step": 6570 + }, + { + "epoch": 0.6798925397809464, + "grad_norm": 1.306950569152832, + "learning_rate": 0.00011827103724317426, + "loss": 0.8071, + "step": 6580 + }, + { + "epoch": 0.6809258111179995, + "grad_norm": 1.175700306892395, + "learning_rate": 0.00011757503063599436, + "loss": 0.7327, + "step": 6590 + }, + { + "epoch": 0.6819590824550527, + "grad_norm": 1.052412509918213, + "learning_rate": 0.00011688044782412124, + "loss": 0.7375, + "step": 6600 + }, + { + "epoch": 0.6829923537921058, + "grad_norm": 0.922924280166626, + "learning_rate": 0.00011618729627551037, + "loss": 0.7765, + "step": 6610 + }, + { + "epoch": 0.6840256251291589, + "grad_norm": 0.6624295711517334, + "learning_rate": 0.00011549558344272834, + "loss": 0.7365, + "step": 6620 + }, + { + "epoch": 0.685058896466212, + "grad_norm": 0.9047833681106567, + "learning_rate": 0.00011480531676287322, + "loss": 0.7549, + "step": 6630 + }, + { + "epoch": 0.6860921678032651, + "grad_norm": 0.8293854594230652, + "learning_rate": 0.00011411650365749454, + "loss": 0.7531, + "step": 6640 + }, + { + "epoch": 0.6871254391403182, + "grad_norm": 0.865106463432312, + "learning_rate": 0.00011342915153251332, + "loss": 0.6403, + "step": 6650 + }, + { + "epoch": 0.6881587104773713, + "grad_norm": 1.0484962463378906, + "learning_rate": 0.0001127432677781425, + "loss": 0.7709, + "step": 6660 + }, + { + "epoch": 0.6891919818144244, + "grad_norm": 0.9296818375587463, + "learning_rate": 0.00011205885976880753, + "loss": 0.6943, + "step": 6670 + }, + { + "epoch": 0.6902252531514775, + "grad_norm": 0.9692133069038391, + "learning_rate": 0.00011137593486306727, + "loss": 0.6684, + "step": 6680 + }, + { + "epoch": 0.6912585244885306, + "grad_norm": 1.2010059356689453, + "learning_rate": 0.00011069450040353448, + "loss": 0.8146, + "step": 6690 + }, + { + "epoch": 0.6922917958255838, + "grad_norm": 0.8989507555961609, + "learning_rate": 0.00011001456371679707, + "loss": 0.6553, + "step": 6700 + }, + { + "epoch": 0.6933250671626369, + "grad_norm": 1.166821837425232, + "learning_rate": 0.00010933613211333943, + "loss": 0.76, + "step": 6710 + }, + { + "epoch": 0.69435833849969, + "grad_norm": 0.8655950427055359, + "learning_rate": 0.00010865921288746362, + "loss": 0.7237, + "step": 6720 + }, + { + "epoch": 0.6953916098367431, + "grad_norm": 1.0120173692703247, + "learning_rate": 0.00010798381331721108, + "loss": 0.8003, + "step": 6730 + }, + { + "epoch": 0.6964248811737962, + "grad_norm": 1.4091392755508423, + "learning_rate": 0.0001073099406642844, + "loss": 0.8101, + "step": 6740 + }, + { + "epoch": 0.6974581525108493, + "grad_norm": 0.8245342969894409, + "learning_rate": 0.00010663760217396906, + "loss": 0.6408, + "step": 6750 + }, + { + "epoch": 0.6984914238479024, + "grad_norm": 0.7788679003715515, + "learning_rate": 0.00010596680507505596, + "loss": 0.7559, + "step": 6760 + }, + { + "epoch": 0.6995246951849555, + "grad_norm": 0.5751708745956421, + "learning_rate": 0.00010529755657976315, + "loss": 0.6268, + "step": 6770 + }, + { + "epoch": 0.7005579665220086, + "grad_norm": 0.9208303689956665, + "learning_rate": 0.00010462986388365837, + "loss": 0.7369, + "step": 6780 + }, + { + "epoch": 0.7015912378590617, + "grad_norm": 0.8211421966552734, + "learning_rate": 0.00010396373416558233, + "loss": 0.7131, + "step": 6790 + }, + { + "epoch": 0.7026245091961149, + "grad_norm": 0.6540753245353699, + "learning_rate": 0.00010329917458757068, + "loss": 0.7293, + "step": 6800 + }, + { + "epoch": 0.703657780533168, + "grad_norm": 1.3494755029678345, + "learning_rate": 0.0001026361922947775, + "loss": 0.6745, + "step": 6810 + }, + { + "epoch": 0.7046910518702211, + "grad_norm": 0.8039237856864929, + "learning_rate": 0.00010197479441539827, + "loss": 0.6993, + "step": 6820 + }, + { + "epoch": 0.7057243232072742, + "grad_norm": 0.8956775069236755, + "learning_rate": 0.00010131498806059352, + "loss": 0.7667, + "step": 6830 + }, + { + "epoch": 0.7067575945443273, + "grad_norm": 0.7461487650871277, + "learning_rate": 0.00010065678032441208, + "loss": 0.6487, + "step": 6840 + }, + { + "epoch": 0.7077908658813804, + "grad_norm": 1.2220842838287354, + "learning_rate": 0.00010000017828371458, + "loss": 0.6455, + "step": 6850 + }, + { + "epoch": 0.7088241372184335, + "grad_norm": 0.9173330068588257, + "learning_rate": 9.934518899809799e-05, + "loss": 0.6879, + "step": 6860 + }, + { + "epoch": 0.7098574085554866, + "grad_norm": 0.6598500609397888, + "learning_rate": 9.869181950981948e-05, + "loss": 0.7175, + "step": 6870 + }, + { + "epoch": 0.7108906798925397, + "grad_norm": 1.0425814390182495, + "learning_rate": 9.804007684372038e-05, + "loss": 0.7662, + "step": 6880 + }, + { + "epoch": 0.7119239512295928, + "grad_norm": 1.0084232091903687, + "learning_rate": 9.738996800715094e-05, + "loss": 0.7261, + "step": 6890 + }, + { + "epoch": 0.712957222566646, + "grad_norm": 0.9493532180786133, + "learning_rate": 9.674149998989523e-05, + "loss": 0.7575, + "step": 6900 + }, + { + "epoch": 0.7139904939036991, + "grad_norm": 0.6258692741394043, + "learning_rate": 9.609467976409525e-05, + "loss": 0.6136, + "step": 6910 + }, + { + "epoch": 0.7150237652407522, + "grad_norm": 0.6811854243278503, + "learning_rate": 9.544951428417667e-05, + "loss": 0.8021, + "step": 6920 + }, + { + "epoch": 0.7160570365778053, + "grad_norm": 0.8507707118988037, + "learning_rate": 9.480601048677371e-05, + "loss": 0.644, + "step": 6930 + }, + { + "epoch": 0.7170903079148584, + "grad_norm": 1.1403719186782837, + "learning_rate": 9.416417529065488e-05, + "loss": 0.7257, + "step": 6940 + }, + { + "epoch": 0.7181235792519115, + "grad_norm": 0.9672065377235413, + "learning_rate": 9.352401559664817e-05, + "loss": 0.6545, + "step": 6950 + }, + { + "epoch": 0.7191568505889646, + "grad_norm": 1.221876859664917, + "learning_rate": 9.288553828756702e-05, + "loss": 0.6231, + "step": 6960 + }, + { + "epoch": 0.7201901219260177, + "grad_norm": 0.8762854337692261, + "learning_rate": 9.224875022813645e-05, + "loss": 0.7172, + "step": 6970 + }, + { + "epoch": 0.7212233932630708, + "grad_norm": 0.811069667339325, + "learning_rate": 9.161365826491913e-05, + "loss": 0.6319, + "step": 6980 + }, + { + "epoch": 0.7222566646001239, + "grad_norm": 1.3847732543945312, + "learning_rate": 9.098026922624175e-05, + "loss": 0.8109, + "step": 6990 + }, + { + "epoch": 0.723289935937177, + "grad_norm": 1.142921805381775, + "learning_rate": 9.03485899221216e-05, + "loss": 0.7571, + "step": 7000 + }, + { + "epoch": 0.7243232072742302, + "grad_norm": 0.5236563086509705, + "learning_rate": 8.971862714419368e-05, + "loss": 0.6592, + "step": 7010 + }, + { + "epoch": 0.7253564786112833, + "grad_norm": 1.2576050758361816, + "learning_rate": 8.909038766563703e-05, + "loss": 0.7714, + "step": 7020 + }, + { + "epoch": 0.7263897499483364, + "grad_norm": 0.9121869802474976, + "learning_rate": 8.846387824110247e-05, + "loss": 0.8312, + "step": 7030 + }, + { + "epoch": 0.7274230212853895, + "grad_norm": 1.3254677057266235, + "learning_rate": 8.783910560663968e-05, + "loss": 0.7242, + "step": 7040 + }, + { + "epoch": 0.7284562926224426, + "grad_norm": 1.342081904411316, + "learning_rate": 8.721607647962496e-05, + "loss": 0.7367, + "step": 7050 + }, + { + "epoch": 0.7294895639594957, + "grad_norm": 1.3918049335479736, + "learning_rate": 8.659479755868882e-05, + "loss": 0.8162, + "step": 7060 + }, + { + "epoch": 0.7305228352965488, + "grad_norm": 0.9307500720024109, + "learning_rate": 8.597527552364415e-05, + "loss": 0.6592, + "step": 7070 + }, + { + "epoch": 0.7315561066336019, + "grad_norm": 0.958733081817627, + "learning_rate": 8.535751703541417e-05, + "loss": 0.7096, + "step": 7080 + }, + { + "epoch": 0.732589377970655, + "grad_norm": 0.6930942535400391, + "learning_rate": 8.474152873596122e-05, + "loss": 0.6513, + "step": 7090 + }, + { + "epoch": 0.7336226493077082, + "grad_norm": 1.3499219417572021, + "learning_rate": 8.412731724821482e-05, + "loss": 0.7495, + "step": 7100 + }, + { + "epoch": 0.7346559206447613, + "grad_norm": 0.9743627905845642, + "learning_rate": 8.351488917600077e-05, + "loss": 0.6989, + "step": 7110 + }, + { + "epoch": 0.7356891919818144, + "grad_norm": 0.8228883147239685, + "learning_rate": 8.290425110397018e-05, + "loss": 0.7556, + "step": 7120 + }, + { + "epoch": 0.7367224633188675, + "grad_norm": 0.9745365381240845, + "learning_rate": 8.229540959752851e-05, + "loss": 0.6756, + "step": 7130 + }, + { + "epoch": 0.7377557346559206, + "grad_norm": 0.9700762629508972, + "learning_rate": 8.168837120276512e-05, + "loss": 0.7298, + "step": 7140 + }, + { + "epoch": 0.7387890059929737, + "grad_norm": 1.2264478206634521, + "learning_rate": 8.108314244638267e-05, + "loss": 0.7307, + "step": 7150 + }, + { + "epoch": 0.7398222773300268, + "grad_norm": 1.1239992380142212, + "learning_rate": 8.047972983562748e-05, + "loss": 0.7043, + "step": 7160 + }, + { + "epoch": 0.7408555486670799, + "grad_norm": 0.8346306681632996, + "learning_rate": 7.987813985821882e-05, + "loss": 0.7194, + "step": 7170 + }, + { + "epoch": 0.741888820004133, + "grad_norm": 0.8321127891540527, + "learning_rate": 7.927837898227966e-05, + "loss": 0.7095, + "step": 7180 + }, + { + "epoch": 0.7429220913411861, + "grad_norm": 0.9774494767189026, + "learning_rate": 7.868045365626702e-05, + "loss": 0.7328, + "step": 7190 + }, + { + "epoch": 0.7439553626782393, + "grad_norm": 0.8818146586418152, + "learning_rate": 7.808437030890258e-05, + "loss": 0.6084, + "step": 7200 + }, + { + "epoch": 0.7449886340152924, + "grad_norm": 0.6830877661705017, + "learning_rate": 7.749013534910362e-05, + "loss": 0.6242, + "step": 7210 + }, + { + "epoch": 0.7460219053523455, + "grad_norm": 0.7373770475387573, + "learning_rate": 7.689775516591397e-05, + "loss": 0.6592, + "step": 7220 + }, + { + "epoch": 0.7470551766893986, + "grad_norm": 1.5116766691207886, + "learning_rate": 7.63072361284357e-05, + "loss": 0.7781, + "step": 7230 + }, + { + "epoch": 0.7480884480264517, + "grad_norm": 1.11928129196167, + "learning_rate": 7.571858458576017e-05, + "loss": 0.8028, + "step": 7240 + }, + { + "epoch": 0.7491217193635048, + "grad_norm": 0.5922040939331055, + "learning_rate": 7.513180686690002e-05, + "loss": 0.6601, + "step": 7250 + }, + { + "epoch": 0.750154990700558, + "grad_norm": 0.9148753881454468, + "learning_rate": 7.454690928072088e-05, + "loss": 0.7152, + "step": 7260 + }, + { + "epoch": 0.7511882620376111, + "grad_norm": 0.7024368047714233, + "learning_rate": 7.396389811587415e-05, + "loss": 0.6906, + "step": 7270 + }, + { + "epoch": 0.7522215333746642, + "grad_norm": 1.3108892440795898, + "learning_rate": 7.338277964072864e-05, + "loss": 0.7471, + "step": 7280 + }, + { + "epoch": 0.7532548047117174, + "grad_norm": 0.6525394320487976, + "learning_rate": 7.280356010330361e-05, + "loss": 0.7364, + "step": 7290 + }, + { + "epoch": 0.7542880760487705, + "grad_norm": 1.344499111175537, + "learning_rate": 7.222624573120137e-05, + "loss": 0.7023, + "step": 7300 + }, + { + "epoch": 0.7553213473858236, + "grad_norm": 0.7209261059761047, + "learning_rate": 7.165084273154074e-05, + "loss": 0.6748, + "step": 7310 + }, + { + "epoch": 0.7563546187228767, + "grad_norm": 1.4905532598495483, + "learning_rate": 7.10773572908898e-05, + "loss": 0.6376, + "step": 7320 + }, + { + "epoch": 0.7573878900599298, + "grad_norm": 0.4700772762298584, + "learning_rate": 7.050579557519938e-05, + "loss": 0.6694, + "step": 7330 + }, + { + "epoch": 0.7584211613969829, + "grad_norm": 0.6527320146560669, + "learning_rate": 6.993616372973739e-05, + "loss": 0.6764, + "step": 7340 + }, + { + "epoch": 0.759454432734036, + "grad_norm": 0.7010611295700073, + "learning_rate": 6.936846787902207e-05, + "loss": 0.658, + "step": 7350 + }, + { + "epoch": 0.7604877040710891, + "grad_norm": 1.1044212579727173, + "learning_rate": 6.880271412675645e-05, + "loss": 0.6569, + "step": 7360 + }, + { + "epoch": 0.7615209754081422, + "grad_norm": 0.8779295682907104, + "learning_rate": 6.823890855576261e-05, + "loss": 0.6382, + "step": 7370 + }, + { + "epoch": 0.7625542467451953, + "grad_norm": 0.9512449502944946, + "learning_rate": 6.76770572279167e-05, + "loss": 0.6453, + "step": 7380 + }, + { + "epoch": 0.7635875180822485, + "grad_norm": 0.782164990901947, + "learning_rate": 6.711716618408281e-05, + "loss": 0.6539, + "step": 7390 + }, + { + "epoch": 0.7646207894193016, + "grad_norm": 1.7972410917282104, + "learning_rate": 6.655924144404906e-05, + "loss": 0.6142, + "step": 7400 + }, + { + "epoch": 0.7656540607563547, + "grad_norm": 0.9399009346961975, + "learning_rate": 6.600328900646222e-05, + "loss": 0.7511, + "step": 7410 + }, + { + "epoch": 0.7666873320934078, + "grad_norm": 2.6008989810943604, + "learning_rate": 6.544931484876368e-05, + "loss": 0.5901, + "step": 7420 + }, + { + "epoch": 0.7677206034304609, + "grad_norm": 1.3178836107254028, + "learning_rate": 6.489732492712466e-05, + "loss": 0.6217, + "step": 7430 + }, + { + "epoch": 0.768753874767514, + "grad_norm": 1.1651490926742554, + "learning_rate": 6.434732517638254e-05, + "loss": 0.6472, + "step": 7440 + }, + { + "epoch": 0.7697871461045671, + "grad_norm": 1.072582721710205, + "learning_rate": 6.37993215099771e-05, + "loss": 0.6595, + "step": 7450 + }, + { + "epoch": 0.7708204174416202, + "grad_norm": 1.5162521600723267, + "learning_rate": 6.325331981988653e-05, + "loss": 0.7025, + "step": 7460 + }, + { + "epoch": 0.7718536887786733, + "grad_norm": 0.804498016834259, + "learning_rate": 6.27093259765645e-05, + "loss": 0.7168, + "step": 7470 + }, + { + "epoch": 0.7728869601157264, + "grad_norm": 1.0188984870910645, + "learning_rate": 6.216734582887678e-05, + "loss": 0.6158, + "step": 7480 + }, + { + "epoch": 0.7739202314527796, + "grad_norm": 0.9018194675445557, + "learning_rate": 6.162738520403871e-05, + "loss": 0.5752, + "step": 7490 + }, + { + "epoch": 0.7749535027898327, + "grad_norm": 0.6925551891326904, + "learning_rate": 6.108944990755202e-05, + "loss": 0.6934, + "step": 7500 + }, + { + "epoch": 0.7759867741268858, + "grad_norm": 1.3633153438568115, + "learning_rate": 6.0553545723142864e-05, + "loss": 0.6985, + "step": 7510 + }, + { + "epoch": 0.7770200454639389, + "grad_norm": 1.114071249961853, + "learning_rate": 6.001967841269937e-05, + "loss": 0.6031, + "step": 7520 + }, + { + "epoch": 0.778053316800992, + "grad_norm": 0.7959581613540649, + "learning_rate": 5.9487853716209794e-05, + "loss": 0.6645, + "step": 7530 + }, + { + "epoch": 0.7790865881380451, + "grad_norm": 0.8598949313163757, + "learning_rate": 5.89580773517008e-05, + "loss": 0.7458, + "step": 7540 + }, + { + "epoch": 0.7801198594750982, + "grad_norm": 1.4160740375518799, + "learning_rate": 5.843035501517596e-05, + "loss": 0.6635, + "step": 7550 + }, + { + "epoch": 0.7811531308121513, + "grad_norm": 1.0915048122406006, + "learning_rate": 5.790469238055465e-05, + "loss": 0.6052, + "step": 7560 + }, + { + "epoch": 0.7821864021492044, + "grad_norm": 0.8317732214927673, + "learning_rate": 5.738109509961076e-05, + "loss": 0.7007, + "step": 7570 + }, + { + "epoch": 0.7832196734862575, + "grad_norm": 0.9893842339515686, + "learning_rate": 5.685956880191218e-05, + "loss": 0.5915, + "step": 7580 + }, + { + "epoch": 0.7842529448233106, + "grad_norm": 1.2170053720474243, + "learning_rate": 5.634011909476008e-05, + "loss": 0.6921, + "step": 7590 + }, + { + "epoch": 0.7852862161603638, + "grad_norm": 1.261479377746582, + "learning_rate": 5.582275156312885e-05, + "loss": 0.6338, + "step": 7600 + }, + { + "epoch": 0.7863194874974169, + "grad_norm": 1.0150582790374756, + "learning_rate": 5.5307471769605875e-05, + "loss": 0.637, + "step": 7610 + }, + { + "epoch": 0.78735275883447, + "grad_norm": 1.148558497428894, + "learning_rate": 5.479428525433167e-05, + "loss": 0.6243, + "step": 7620 + }, + { + "epoch": 0.7883860301715231, + "grad_norm": 0.6112420558929443, + "learning_rate": 5.4283197534940684e-05, + "loss": 0.579, + "step": 7630 + }, + { + "epoch": 0.7894193015085762, + "grad_norm": 1.1648898124694824, + "learning_rate": 5.377421410650149e-05, + "loss": 0.6978, + "step": 7640 + }, + { + "epoch": 0.7904525728456293, + "grad_norm": 1.3245562314987183, + "learning_rate": 5.326734044145801e-05, + "loss": 0.5724, + "step": 7650 + }, + { + "epoch": 0.7914858441826824, + "grad_norm": 1.0554862022399902, + "learning_rate": 5.276258198957051e-05, + "loss": 0.7884, + "step": 7660 + }, + { + "epoch": 0.7925191155197355, + "grad_norm": 1.1452159881591797, + "learning_rate": 5.225994417785726e-05, + "loss": 0.7309, + "step": 7670 + }, + { + "epoch": 0.7935523868567886, + "grad_norm": 0.7298494577407837, + "learning_rate": 5.175943241053582e-05, + "loss": 0.6804, + "step": 7680 + }, + { + "epoch": 0.7945856581938417, + "grad_norm": 1.0335783958435059, + "learning_rate": 5.1261052068965306e-05, + "loss": 0.6306, + "step": 7690 + }, + { + "epoch": 0.7956189295308949, + "grad_norm": 0.6662817001342773, + "learning_rate": 5.0764808511588155e-05, + "loss": 0.6118, + "step": 7700 + }, + { + "epoch": 0.796652200867948, + "grad_norm": 0.8202494978904724, + "learning_rate": 5.027070707387296e-05, + "loss": 0.6764, + "step": 7710 + }, + { + "epoch": 0.7976854722050011, + "grad_norm": 1.5033270120620728, + "learning_rate": 4.977875306825672e-05, + "loss": 0.7725, + "step": 7720 + }, + { + "epoch": 0.7987187435420542, + "grad_norm": 0.7202991843223572, + "learning_rate": 4.928895178408782e-05, + "loss": 0.685, + "step": 7730 + }, + { + "epoch": 0.7997520148791073, + "grad_norm": 0.8215998411178589, + "learning_rate": 4.880130848756925e-05, + "loss": 0.6536, + "step": 7740 + }, + { + "epoch": 0.8007852862161604, + "grad_norm": 1.0507868528366089, + "learning_rate": 4.8315828421701986e-05, + "loss": 0.6238, + "step": 7750 + }, + { + "epoch": 0.8018185575532135, + "grad_norm": 1.0970897674560547, + "learning_rate": 4.783251680622847e-05, + "loss": 0.6302, + "step": 7760 + }, + { + "epoch": 0.8028518288902666, + "grad_norm": 1.2945756912231445, + "learning_rate": 4.7351378837576565e-05, + "loss": 0.5465, + "step": 7770 + }, + { + "epoch": 0.8038851002273197, + "grad_norm": 1.6131089925765991, + "learning_rate": 4.687241968880393e-05, + "loss": 0.6698, + "step": 7780 + }, + { + "epoch": 0.8049183715643728, + "grad_norm": 1.0525102615356445, + "learning_rate": 4.639564450954201e-05, + "loss": 0.6425, + "step": 7790 + }, + { + "epoch": 0.805951642901426, + "grad_norm": 1.0598100423812866, + "learning_rate": 4.5921058425940745e-05, + "loss": 0.5235, + "step": 7800 + }, + { + "epoch": 0.8069849142384791, + "grad_norm": 1.4403691291809082, + "learning_rate": 4.544866654061364e-05, + "loss": 0.7243, + "step": 7810 + }, + { + "epoch": 0.8080181855755322, + "grad_norm": 0.995022714138031, + "learning_rate": 4.497847393258292e-05, + "loss": 0.6382, + "step": 7820 + }, + { + "epoch": 0.8090514569125853, + "grad_norm": 1.2074267864227295, + "learning_rate": 4.451048565722468e-05, + "loss": 0.644, + "step": 7830 + }, + { + "epoch": 0.8100847282496384, + "grad_norm": 1.038830041885376, + "learning_rate": 4.404470674621463e-05, + "loss": 0.6955, + "step": 7840 + }, + { + "epoch": 0.8111179995866915, + "grad_norm": 1.028713345527649, + "learning_rate": 4.3581142207474225e-05, + "loss": 0.5929, + "step": 7850 + }, + { + "epoch": 0.8121512709237446, + "grad_norm": 1.214220404624939, + "learning_rate": 4.311979702511645e-05, + "loss": 0.6807, + "step": 7860 + }, + { + "epoch": 0.8131845422607977, + "grad_norm": 0.91230708360672, + "learning_rate": 4.266067615939234e-05, + "loss": 0.5622, + "step": 7870 + }, + { + "epoch": 0.8142178135978508, + "grad_norm": 0.7656465172767639, + "learning_rate": 4.220378454663784e-05, + "loss": 0.7191, + "step": 7880 + }, + { + "epoch": 0.815251084934904, + "grad_norm": 0.9677980542182922, + "learning_rate": 4.1749127099220684e-05, + "loss": 0.6268, + "step": 7890 + }, + { + "epoch": 0.8162843562719571, + "grad_norm": 0.8608049750328064, + "learning_rate": 4.129670870548738e-05, + "loss": 0.608, + "step": 7900 + }, + { + "epoch": 0.8173176276090102, + "grad_norm": 1.3923230171203613, + "learning_rate": 4.084653422971077e-05, + "loss": 0.6626, + "step": 7910 + }, + { + "epoch": 0.8183508989460633, + "grad_norm": 1.0703678131103516, + "learning_rate": 4.039860851203775e-05, + "loss": 0.5827, + "step": 7920 + }, + { + "epoch": 0.8193841702831164, + "grad_norm": 0.9771378040313721, + "learning_rate": 3.9952936368437275e-05, + "loss": 0.7703, + "step": 7930 + }, + { + "epoch": 0.8204174416201695, + "grad_norm": 1.0725332498550415, + "learning_rate": 3.950952259064841e-05, + "loss": 0.6425, + "step": 7940 + }, + { + "epoch": 0.8214507129572226, + "grad_norm": 1.1629067659378052, + "learning_rate": 3.9068371946128945e-05, + "loss": 0.6658, + "step": 7950 + }, + { + "epoch": 0.8224839842942757, + "grad_norm": 0.9839202165603638, + "learning_rate": 3.862948917800424e-05, + "loss": 0.5883, + "step": 7960 + }, + { + "epoch": 0.8235172556313288, + "grad_norm": 0.8677191138267517, + "learning_rate": 3.8192879005015974e-05, + "loss": 0.6664, + "step": 7970 + }, + { + "epoch": 0.8245505269683819, + "grad_norm": 0.8415852189064026, + "learning_rate": 3.775854612147153e-05, + "loss": 0.5629, + "step": 7980 + }, + { + "epoch": 0.825583798305435, + "grad_norm": 0.7151034474372864, + "learning_rate": 3.7326495197193555e-05, + "loss": 0.5575, + "step": 7990 + }, + { + "epoch": 0.8266170696424882, + "grad_norm": 1.2307301759719849, + "learning_rate": 3.689673087746975e-05, + "loss": 0.6356, + "step": 8000 + }, + { + "epoch": 0.8276503409795413, + "grad_norm": 2.110441207885742, + "learning_rate": 3.646925778300286e-05, + "loss": 0.7123, + "step": 8010 + }, + { + "epoch": 0.8286836123165944, + "grad_norm": 0.47642096877098083, + "learning_rate": 3.604408050986105e-05, + "loss": 0.6165, + "step": 8020 + }, + { + "epoch": 0.8297168836536475, + "grad_norm": 0.69282066822052, + "learning_rate": 3.5621203629428386e-05, + "loss": 0.6196, + "step": 8030 + }, + { + "epoch": 0.8307501549907006, + "grad_norm": 1.5054296255111694, + "learning_rate": 3.520063168835605e-05, + "loss": 0.749, + "step": 8040 + }, + { + "epoch": 0.8317834263277537, + "grad_norm": 1.0589144229888916, + "learning_rate": 3.478236920851283e-05, + "loss": 0.6056, + "step": 8050 + }, + { + "epoch": 0.8328166976648068, + "grad_norm": 0.8430522680282593, + "learning_rate": 3.4366420686937044e-05, + "loss": 0.6502, + "step": 8060 + }, + { + "epoch": 0.8338499690018599, + "grad_norm": 0.8304775357246399, + "learning_rate": 3.395279059578798e-05, + "loss": 0.6363, + "step": 8070 + }, + { + "epoch": 0.834883240338913, + "grad_norm": 0.5285193920135498, + "learning_rate": 3.3541483382297734e-05, + "loss": 0.751, + "step": 8080 + }, + { + "epoch": 0.8359165116759661, + "grad_norm": 0.9843568205833435, + "learning_rate": 3.313250346872362e-05, + "loss": 0.5978, + "step": 8090 + }, + { + "epoch": 0.8369497830130193, + "grad_norm": 1.0727510452270508, + "learning_rate": 3.272585525230032e-05, + "loss": 0.528, + "step": 8100 + }, + { + "epoch": 0.8379830543500724, + "grad_norm": 0.9402909874916077, + "learning_rate": 3.232154310519298e-05, + "loss": 0.5649, + "step": 8110 + }, + { + "epoch": 0.8390163256871255, + "grad_norm": 0.6825330257415771, + "learning_rate": 3.1919571374449894e-05, + "loss": 0.6619, + "step": 8120 + }, + { + "epoch": 0.8400495970241786, + "grad_norm": 1.605404019355774, + "learning_rate": 3.151994438195582e-05, + "loss": 0.7536, + "step": 8130 + }, + { + "epoch": 0.8410828683612317, + "grad_norm": 0.9325007796287537, + "learning_rate": 3.112266642438566e-05, + "loss": 0.7271, + "step": 8140 + }, + { + "epoch": 0.8421161396982848, + "grad_norm": 1.083452820777893, + "learning_rate": 3.072774177315807e-05, + "loss": 0.6664, + "step": 8150 + }, + { + "epoch": 0.8431494110353379, + "grad_norm": 0.6602939963340759, + "learning_rate": 3.0335174674389732e-05, + "loss": 0.5289, + "step": 8160 + }, + { + "epoch": 0.844182682372391, + "grad_norm": 1.497051477432251, + "learning_rate": 2.9944969348849493e-05, + "loss": 0.5557, + "step": 8170 + }, + { + "epoch": 0.8452159537094441, + "grad_norm": 0.7010759115219116, + "learning_rate": 2.9557129991913235e-05, + "loss": 0.5932, + "step": 8180 + }, + { + "epoch": 0.8462492250464972, + "grad_norm": 0.93887859582901, + "learning_rate": 2.917166077351846e-05, + "loss": 0.5421, + "step": 8190 + }, + { + "epoch": 0.8472824963835504, + "grad_norm": 1.242521047592163, + "learning_rate": 2.8788565838119757e-05, + "loss": 0.5551, + "step": 8200 + }, + { + "epoch": 0.8483157677206035, + "grad_norm": 1.1588674783706665, + "learning_rate": 2.840784930464399e-05, + "loss": 0.5647, + "step": 8210 + }, + { + "epoch": 0.8493490390576566, + "grad_norm": 1.381052851676941, + "learning_rate": 2.8029515266446232e-05, + "loss": 0.6187, + "step": 8220 + }, + { + "epoch": 0.8503823103947097, + "grad_norm": 1.1169166564941406, + "learning_rate": 2.765356779126557e-05, + "loss": 0.6038, + "step": 8230 + }, + { + "epoch": 0.8514155817317628, + "grad_norm": 1.2974213361740112, + "learning_rate": 2.7280010921181487e-05, + "loss": 0.7017, + "step": 8240 + }, + { + "epoch": 0.8524488530688159, + "grad_norm": 0.9372439384460449, + "learning_rate": 2.690884867257032e-05, + "loss": 0.5887, + "step": 8250 + }, + { + "epoch": 0.853482124405869, + "grad_norm": 0.8471134305000305, + "learning_rate": 2.6540085036062227e-05, + "loss": 0.5159, + "step": 8260 + }, + { + "epoch": 0.8545153957429221, + "grad_norm": 1.077082633972168, + "learning_rate": 2.617372397649814e-05, + "loss": 0.6389, + "step": 8270 + }, + { + "epoch": 0.8555486670799752, + "grad_norm": 1.1442515850067139, + "learning_rate": 2.5809769432886986e-05, + "loss": 0.6233, + "step": 8280 + }, + { + "epoch": 0.8565819384170283, + "grad_norm": 1.59578537940979, + "learning_rate": 2.5448225318363828e-05, + "loss": 0.6336, + "step": 8290 + }, + { + "epoch": 0.8576152097540815, + "grad_norm": 0.7789118885993958, + "learning_rate": 2.5089095520147266e-05, + "loss": 0.6106, + "step": 8300 + }, + { + "epoch": 0.8586484810911346, + "grad_norm": 0.8877026438713074, + "learning_rate": 2.4732383899497972e-05, + "loss": 0.6226, + "step": 8310 + }, + { + "epoch": 0.8596817524281877, + "grad_norm": 0.9339987635612488, + "learning_rate": 2.4378094291676978e-05, + "loss": 0.5998, + "step": 8320 + }, + { + "epoch": 0.8607150237652408, + "grad_norm": 0.45565560460090637, + "learning_rate": 2.4026230505904633e-05, + "loss": 0.5463, + "step": 8330 + }, + { + "epoch": 0.8617482951022939, + "grad_norm": 0.7657277584075928, + "learning_rate": 2.367679632531955e-05, + "loss": 0.6507, + "step": 8340 + }, + { + "epoch": 0.862781566439347, + "grad_norm": 1.4094293117523193, + "learning_rate": 2.3329795506937728e-05, + "loss": 0.5541, + "step": 8350 + }, + { + "epoch": 0.8638148377764001, + "grad_norm": 0.5916699171066284, + "learning_rate": 2.2985231781612466e-05, + "loss": 0.6416, + "step": 8360 + }, + { + "epoch": 0.8648481091134532, + "grad_norm": 0.7877978086471558, + "learning_rate": 2.2643108853994317e-05, + "loss": 0.6413, + "step": 8370 + }, + { + "epoch": 0.8658813804505063, + "grad_norm": 1.3543034791946411, + "learning_rate": 2.2303430402490805e-05, + "loss": 0.6255, + "step": 8380 + }, + { + "epoch": 0.8669146517875594, + "grad_norm": 1.2339602708816528, + "learning_rate": 2.1966200079227257e-05, + "loss": 0.6086, + "step": 8390 + }, + { + "epoch": 0.8679479231246126, + "grad_norm": 1.563103199005127, + "learning_rate": 2.1631421510007575e-05, + "loss": 0.6084, + "step": 8400 + }, + { + "epoch": 0.8689811944616657, + "grad_norm": 1.1265037059783936, + "learning_rate": 2.129909829427479e-05, + "loss": 0.5604, + "step": 8410 + }, + { + "epoch": 0.8700144657987188, + "grad_norm": 1.4551678895950317, + "learning_rate": 2.0969234005072916e-05, + "loss": 0.6835, + "step": 8420 + }, + { + "epoch": 0.8710477371357719, + "grad_norm": 0.9663675427436829, + "learning_rate": 2.064183218900817e-05, + "loss": 0.6, + "step": 8430 + }, + { + "epoch": 0.872081008472825, + "grad_norm": 1.2805395126342773, + "learning_rate": 2.0316896366211118e-05, + "loss": 0.6471, + "step": 8440 + }, + { + "epoch": 0.8731142798098781, + "grad_norm": 0.9442639946937561, + "learning_rate": 1.9994430030298496e-05, + "loss": 0.6104, + "step": 8450 + }, + { + "epoch": 0.8741475511469312, + "grad_norm": 1.5761253833770752, + "learning_rate": 1.967443664833596e-05, + "loss": 0.6629, + "step": 8460 + }, + { + "epoch": 0.8751808224839843, + "grad_norm": 1.1689879894256592, + "learning_rate": 1.9356919660800553e-05, + "loss": 0.6161, + "step": 8470 + }, + { + "epoch": 0.8762140938210374, + "grad_norm": 1.1315240859985352, + "learning_rate": 1.9041882481543955e-05, + "loss": 0.574, + "step": 8480 + }, + { + "epoch": 0.8772473651580905, + "grad_norm": 1.0627079010009766, + "learning_rate": 1.8729328497755578e-05, + "loss": 0.6021, + "step": 8490 + }, + { + "epoch": 0.8782806364951437, + "grad_norm": 0.9222522377967834, + "learning_rate": 1.8419261069926197e-05, + "loss": 0.5026, + "step": 8500 + }, + { + "epoch": 0.8793139078321968, + "grad_norm": 0.92143315076828, + "learning_rate": 1.8111683531812002e-05, + "loss": 0.5853, + "step": 8510 + }, + { + "epoch": 0.8803471791692499, + "grad_norm": 1.042581558227539, + "learning_rate": 1.7806599190398455e-05, + "loss": 0.5239, + "step": 8520 + }, + { + "epoch": 0.881380450506303, + "grad_norm": 1.405364751815796, + "learning_rate": 1.750401132586496e-05, + "loss": 0.571, + "step": 8530 + }, + { + "epoch": 0.8824137218433561, + "grad_norm": 1.2399895191192627, + "learning_rate": 1.720392319154948e-05, + "loss": 0.5704, + "step": 8540 + }, + { + "epoch": 0.8834469931804092, + "grad_norm": 0.9870766401290894, + "learning_rate": 1.6906338013913585e-05, + "loss": 0.5944, + "step": 8550 + }, + { + "epoch": 0.8844802645174623, + "grad_norm": 0.8951932787895203, + "learning_rate": 1.6611258992507817e-05, + "loss": 0.5861, + "step": 8560 + }, + { + "epoch": 0.8855135358545154, + "grad_norm": 1.8214749097824097, + "learning_rate": 1.6318689299937213e-05, + "loss": 0.6643, + "step": 8570 + }, + { + "epoch": 0.8865468071915685, + "grad_norm": 0.8029395341873169, + "learning_rate": 1.6028632081827173e-05, + "loss": 0.5645, + "step": 8580 + }, + { + "epoch": 0.8875800785286216, + "grad_norm": 0.43516650795936584, + "learning_rate": 1.574109045678987e-05, + "loss": 0.5996, + "step": 8590 + }, + { + "epoch": 0.8886133498656748, + "grad_norm": 1.673608422279358, + "learning_rate": 1.545606751639034e-05, + "loss": 0.6781, + "step": 8600 + }, + { + "epoch": 0.8896466212027279, + "grad_norm": 1.0479223728179932, + "learning_rate": 1.5173566325113575e-05, + "loss": 0.5522, + "step": 8610 + }, + { + "epoch": 0.890679892539781, + "grad_norm": 1.2102528810501099, + "learning_rate": 1.4893589920331352e-05, + "loss": 0.5926, + "step": 8620 + }, + { + "epoch": 0.8917131638768341, + "grad_norm": 1.0137587785720825, + "learning_rate": 1.4616141312269737e-05, + "loss": 0.5393, + "step": 8630 + }, + { + "epoch": 0.8927464352138872, + "grad_norm": 2.0007264614105225, + "learning_rate": 1.4341223483976624e-05, + "loss": 0.6629, + "step": 8640 + }, + { + "epoch": 0.8937797065509403, + "grad_norm": 0.966791570186615, + "learning_rate": 1.406883939128964e-05, + "loss": 0.6255, + "step": 8650 + }, + { + "epoch": 0.8948129778879934, + "grad_norm": 1.436010479927063, + "learning_rate": 1.3798991962804541e-05, + "loss": 0.6226, + "step": 8660 + }, + { + "epoch": 0.8958462492250465, + "grad_norm": 1.060235857963562, + "learning_rate": 1.3531684099843483e-05, + "loss": 0.6685, + "step": 8670 + }, + { + "epoch": 0.8968795205620996, + "grad_norm": 0.578377902507782, + "learning_rate": 1.3266918676423962e-05, + "loss": 0.5681, + "step": 8680 + }, + { + "epoch": 0.8979127918991527, + "grad_norm": 0.5564404726028442, + "learning_rate": 1.300469853922795e-05, + "loss": 0.587, + "step": 8690 + }, + { + "epoch": 0.8989460632362059, + "grad_norm": 0.9987608194351196, + "learning_rate": 1.2745026507571145e-05, + "loss": 0.65, + "step": 8700 + }, + { + "epoch": 0.899979334573259, + "grad_norm": 1.4747377634048462, + "learning_rate": 1.2487905373372799e-05, + "loss": 0.6024, + "step": 8710 + }, + { + "epoch": 0.9010126059103121, + "grad_norm": 0.7746207118034363, + "learning_rate": 1.223333790112563e-05, + "loss": 0.6037, + "step": 8720 + }, + { + "epoch": 0.9020458772473652, + "grad_norm": 1.0953818559646606, + "learning_rate": 1.1981326827866124e-05, + "loss": 0.6469, + "step": 8730 + }, + { + "epoch": 0.9030791485844183, + "grad_norm": 1.1232373714447021, + "learning_rate": 1.1731874863145143e-05, + "loss": 0.591, + "step": 8740 + }, + { + "epoch": 0.9041124199214714, + "grad_norm": 2.062878131866455, + "learning_rate": 1.148498468899864e-05, + "loss": 0.6622, + "step": 8750 + }, + { + "epoch": 0.9051456912585245, + "grad_norm": 1.1355712413787842, + "learning_rate": 1.1240658959918965e-05, + "loss": 0.762, + "step": 8760 + }, + { + "epoch": 0.9061789625955776, + "grad_norm": 1.6199957132339478, + "learning_rate": 1.0998900302826382e-05, + "loss": 0.6976, + "step": 8770 + }, + { + "epoch": 0.9072122339326307, + "grad_norm": 1.111836552619934, + "learning_rate": 1.0759711317040649e-05, + "loss": 0.5116, + "step": 8780 + }, + { + "epoch": 0.9082455052696838, + "grad_norm": 1.650025725364685, + "learning_rate": 1.0523094574253101e-05, + "loss": 0.5384, + "step": 8790 + }, + { + "epoch": 0.909278776606737, + "grad_norm": 0.9917317628860474, + "learning_rate": 1.0289052618499218e-05, + "loss": 0.5526, + "step": 8800 + }, + { + "epoch": 0.9103120479437901, + "grad_norm": 0.618635356426239, + "learning_rate": 1.005758796613096e-05, + "loss": 0.6995, + "step": 8810 + }, + { + "epoch": 0.9113453192808432, + "grad_norm": 1.289839506149292, + "learning_rate": 9.828703105789983e-06, + "loss": 0.6453, + "step": 8820 + }, + { + "epoch": 0.9123785906178963, + "grad_norm": 1.1484878063201904, + "learning_rate": 9.602400498380542e-06, + "loss": 0.5949, + "step": 8830 + }, + { + "epoch": 0.9134118619549494, + "grad_norm": 1.1618013381958008, + "learning_rate": 9.378682577043524e-06, + "loss": 0.633, + "step": 8840 + }, + { + "epoch": 0.9144451332920025, + "grad_norm": 0.8447564840316772, + "learning_rate": 9.157551747129844e-06, + "loss": 0.6921, + "step": 8850 + }, + { + "epoch": 0.9154784046290556, + "grad_norm": 2.2064759731292725, + "learning_rate": 8.939010386174783e-06, + "loss": 0.5906, + "step": 8860 + }, + { + "epoch": 0.9165116759661087, + "grad_norm": 1.3931576013565063, + "learning_rate": 8.723060843872393e-06, + "loss": 0.6567, + "step": 8870 + }, + { + "epoch": 0.9175449473031618, + "grad_norm": 0.8289423584938049, + "learning_rate": 8.50970544205032e-06, + "loss": 0.6287, + "step": 8880 + }, + { + "epoch": 0.9185782186402149, + "grad_norm": 0.911371111869812, + "learning_rate": 8.298946474644575e-06, + "loss": 0.541, + "step": 8890 + }, + { + "epoch": 0.919611489977268, + "grad_norm": 1.081002950668335, + "learning_rate": 8.090786207675171e-06, + "loss": 0.6585, + "step": 8900 + }, + { + "epoch": 0.9206447613143212, + "grad_norm": 1.1589564085006714, + "learning_rate": 7.885226879221691e-06, + "loss": 0.644, + "step": 8910 + }, + { + "epoch": 0.9216780326513743, + "grad_norm": 0.6146034002304077, + "learning_rate": 7.682270699399057e-06, + "loss": 0.525, + "step": 8920 + }, + { + "epoch": 0.9227113039884274, + "grad_norm": 0.8697605729103088, + "learning_rate": 7.481919850333946e-06, + "loss": 0.5737, + "step": 8930 + }, + { + "epoch": 0.9237445753254805, + "grad_norm": 1.1086236238479614, + "learning_rate": 7.284176486141214e-06, + "loss": 0.5475, + "step": 8940 + }, + { + "epoch": 0.9247778466625336, + "grad_norm": 1.8016564846038818, + "learning_rate": 7.0890427329008964e-06, + "loss": 0.6598, + "step": 8950 + }, + { + "epoch": 0.9258111179995867, + "grad_norm": 1.1818935871124268, + "learning_rate": 6.896520688635111e-06, + "loss": 0.634, + "step": 8960 + }, + { + "epoch": 0.9268443893366398, + "grad_norm": 1.2800990343093872, + "learning_rate": 6.706612423285685e-06, + "loss": 0.6006, + "step": 8970 + }, + { + "epoch": 0.9278776606736929, + "grad_norm": 1.0055065155029297, + "learning_rate": 6.519319978691845e-06, + "loss": 0.5678, + "step": 8980 + }, + { + "epoch": 0.928910932010746, + "grad_norm": 2.050903081893921, + "learning_rate": 6.334645368568315e-06, + "loss": 0.6205, + "step": 8990 + }, + { + "epoch": 0.9299442033477991, + "grad_norm": 1.3951302766799927, + "learning_rate": 6.152590578483497e-06, + "loss": 0.5148, + "step": 9000 + }, + { + "epoch": 0.9309774746848523, + "grad_norm": 2.197643995285034, + "learning_rate": 5.973157565838327e-06, + "loss": 0.6931, + "step": 9010 + }, + { + "epoch": 0.9320107460219054, + "grad_norm": 1.161935567855835, + "learning_rate": 5.796348259845096e-06, + "loss": 0.5684, + "step": 9020 + }, + { + "epoch": 0.9330440173589585, + "grad_norm": 1.7656304836273193, + "learning_rate": 5.622164561506826e-06, + "loss": 0.6114, + "step": 9030 + }, + { + "epoch": 0.9340772886960116, + "grad_norm": 0.9781146049499512, + "learning_rate": 5.450608343596647e-06, + "loss": 0.5743, + "step": 9040 + }, + { + "epoch": 0.9351105600330647, + "grad_norm": 0.7141128182411194, + "learning_rate": 5.281681450637843e-06, + "loss": 0.5756, + "step": 9050 + }, + { + "epoch": 0.9361438313701178, + "grad_norm": 1.8938722610473633, + "learning_rate": 5.115385698883979e-06, + "loss": 0.5679, + "step": 9060 + }, + { + "epoch": 0.9371771027071709, + "grad_norm": 1.8188812732696533, + "learning_rate": 4.951722876299275e-06, + "loss": 0.744, + "step": 9070 + }, + { + "epoch": 0.938210374044224, + "grad_norm": 0.8902860879898071, + "learning_rate": 4.790694742539487e-06, + "loss": 0.6274, + "step": 9080 + }, + { + "epoch": 0.9392436453812771, + "grad_norm": 2.0708553791046143, + "learning_rate": 4.632303028932943e-06, + "loss": 0.6173, + "step": 9090 + }, + { + "epoch": 0.9402769167183302, + "grad_norm": 1.710466742515564, + "learning_rate": 4.476549438461958e-06, + "loss": 0.6267, + "step": 9100 + }, + { + "epoch": 0.9413101880553834, + "grad_norm": 0.8895155787467957, + "learning_rate": 4.323435645744445e-06, + "loss": 0.7292, + "step": 9110 + }, + { + "epoch": 0.9423434593924365, + "grad_norm": 1.9920309782028198, + "learning_rate": 4.172963297015997e-06, + "loss": 0.6778, + "step": 9120 + }, + { + "epoch": 0.9433767307294896, + "grad_norm": 1.4172946214675903, + "learning_rate": 4.025134010112258e-06, + "loss": 0.5764, + "step": 9130 + }, + { + "epoch": 0.9444100020665427, + "grad_norm": 0.9763116836547852, + "learning_rate": 3.879949374451269e-06, + "loss": 0.5936, + "step": 9140 + }, + { + "epoch": 0.9454432734035958, + "grad_norm": 0.7206150889396667, + "learning_rate": 3.737410951016623e-06, + "loss": 0.6432, + "step": 9150 + }, + { + "epoch": 0.9464765447406489, + "grad_norm": 1.7434405088424683, + "learning_rate": 3.5975202723405874e-06, + "loss": 0.6962, + "step": 9160 + }, + { + "epoch": 0.947509816077702, + "grad_norm": 1.5172473192214966, + "learning_rate": 3.4602788424876464e-06, + "loss": 0.6048, + "step": 9170 + }, + { + "epoch": 0.9485430874147551, + "grad_norm": 1.1150609254837036, + "learning_rate": 3.3256881370383183e-06, + "loss": 0.5865, + "step": 9180 + }, + { + "epoch": 0.9495763587518082, + "grad_norm": 0.8272204995155334, + "learning_rate": 3.1937496030732536e-06, + "loss": 0.7075, + "step": 9190 + }, + { + "epoch": 0.9506096300888613, + "grad_norm": 0.9530496597290039, + "learning_rate": 3.0644646591577174e-06, + "loss": 0.5166, + "step": 9200 + }, + { + "epoch": 0.9516429014259145, + "grad_norm": 0.7508150935173035, + "learning_rate": 2.9378346953264633e-06, + "loss": 0.5569, + "step": 9210 + }, + { + "epoch": 0.9526761727629676, + "grad_norm": 1.5538454055786133, + "learning_rate": 2.8138610730684686e-06, + "loss": 0.6117, + "step": 9220 + }, + { + "epoch": 0.9537094441000207, + "grad_norm": 1.6959922313690186, + "learning_rate": 2.692545125312612e-06, + "loss": 0.5843, + "step": 9230 + }, + { + "epoch": 0.9547427154370738, + "grad_norm": 1.9940367937088013, + "learning_rate": 2.573888156413212e-06, + "loss": 0.6021, + "step": 9240 + }, + { + "epoch": 0.9557759867741269, + "grad_norm": 1.3337304592132568, + "learning_rate": 2.4578914421359288e-06, + "loss": 0.525, + "step": 9250 + }, + { + "epoch": 0.95680925811118, + "grad_norm": 0.8987991213798523, + "learning_rate": 2.344556229644218e-06, + "loss": 0.6227, + "step": 9260 + }, + { + "epoch": 0.9578425294482331, + "grad_norm": 0.8351725339889526, + "learning_rate": 2.233883737485731e-06, + "loss": 0.5019, + "step": 9270 + }, + { + "epoch": 0.9588758007852862, + "grad_norm": 0.8255094289779663, + "learning_rate": 2.1258751555794654e-06, + "loss": 0.6226, + "step": 9280 + }, + { + "epoch": 0.9599090721223393, + "grad_norm": 0.882256269454956, + "learning_rate": 2.020531645202689e-06, + "loss": 0.4691, + "step": 9290 + }, + { + "epoch": 0.9609423434593924, + "grad_norm": 1.3614681959152222, + "learning_rate": 1.9178543389786475e-06, + "loss": 0.6947, + "step": 9300 + }, + { + "epoch": 0.9619756147964456, + "grad_norm": 1.0351027250289917, + "learning_rate": 1.8178443408642386e-06, + "loss": 0.5705, + "step": 9310 + }, + { + "epoch": 0.9630088861334987, + "grad_norm": 1.4347681999206543, + "learning_rate": 1.7205027261383565e-06, + "loss": 0.659, + "step": 9320 + }, + { + "epoch": 0.9640421574705518, + "grad_norm": 0.7792657017707825, + "learning_rate": 1.625830541390122e-06, + "loss": 0.5753, + "step": 9330 + }, + { + "epoch": 0.9650754288076049, + "grad_norm": 2.364046812057495, + "learning_rate": 1.5338288045076697e-06, + "loss": 0.626, + "step": 9340 + }, + { + "epoch": 0.966108700144658, + "grad_norm": 0.6518005728721619, + "learning_rate": 1.4444985046673798e-06, + "loss": 0.5726, + "step": 9350 + }, + { + "epoch": 0.9671419714817111, + "grad_norm": 0.8066127896308899, + "learning_rate": 1.3578406023229962e-06, + "loss": 0.6068, + "step": 9360 + }, + { + "epoch": 0.9681752428187642, + "grad_norm": 0.918552815914154, + "learning_rate": 1.2738560291954415e-06, + "loss": 0.5854, + "step": 9370 + }, + { + "epoch": 0.9692085141558173, + "grad_norm": 0.840164840221405, + "learning_rate": 1.1925456882627417e-06, + "loss": 0.5582, + "step": 9380 + }, + { + "epoch": 0.9702417854928704, + "grad_norm": 1.3730981349945068, + "learning_rate": 1.113910453750394e-06, + "loss": 0.6132, + "step": 9390 + }, + { + "epoch": 0.9712750568299235, + "grad_norm": 0.9575563669204712, + "learning_rate": 1.0379511711219313e-06, + "loss": 0.552, + "step": 9400 + }, + { + "epoch": 0.9723083281669767, + "grad_norm": 1.423699975013733, + "learning_rate": 9.646686570697061e-07, + "loss": 0.6279, + "step": 9410 + }, + { + "epoch": 0.9733415995040298, + "grad_norm": 1.0341185331344604, + "learning_rate": 8.9406369950637e-07, + "loss": 0.6182, + "step": 9420 + }, + { + "epoch": 0.9743748708410829, + "grad_norm": 0.9466649889945984, + "learning_rate": 8.261370575561866e-07, + "loss": 0.6104, + "step": 9430 + }, + { + "epoch": 0.975408142178136, + "grad_norm": 1.4443756341934204, + "learning_rate": 7.608894615468709e-07, + "loss": 0.6099, + "step": 9440 + }, + { + "epoch": 0.9764414135151891, + "grad_norm": 1.057411551475525, + "learning_rate": 6.983216130019288e-07, + "loss": 0.621, + "step": 9450 + }, + { + "epoch": 0.9774746848522422, + "grad_norm": 1.0595217943191528, + "learning_rate": 6.384341846329134e-07, + "loss": 0.6057, + "step": 9460 + }, + { + "epoch": 0.9785079561892953, + "grad_norm": 1.049188494682312, + "learning_rate": 5.812278203322918e-07, + "loss": 0.5873, + "step": 9470 + }, + { + "epoch": 0.9795412275263484, + "grad_norm": 1.212692379951477, + "learning_rate": 5.267031351664786e-07, + "loss": 0.6586, + "step": 9480 + }, + { + "epoch": 0.9805744988634015, + "grad_norm": 0.6824125647544861, + "learning_rate": 4.7486071536925745e-07, + "loss": 0.5165, + "step": 9490 + }, + { + "epoch": 0.9816077702004546, + "grad_norm": 0.6535896062850952, + "learning_rate": 4.257011183354809e-07, + "loss": 0.568, + "step": 9500 + }, + { + "epoch": 0.9826410415375078, + "grad_norm": 1.9366480112075806, + "learning_rate": 3.792248726150471e-07, + "loss": 0.5708, + "step": 9510 + }, + { + "epoch": 0.9836743128745609, + "grad_norm": 0.8946526646614075, + "learning_rate": 3.354324779071827e-07, + "loss": 0.499, + "step": 9520 + }, + { + "epoch": 0.984707584211614, + "grad_norm": 1.2145295143127441, + "learning_rate": 2.9432440505522406e-07, + "loss": 0.6079, + "step": 9530 + }, + { + "epoch": 0.9857408555486671, + "grad_norm": 0.948176383972168, + "learning_rate": 2.559010960413444e-07, + "loss": 0.5659, + "step": 9540 + }, + { + "epoch": 0.9867741268857202, + "grad_norm": 0.8348559141159058, + "learning_rate": 2.201629639819458e-07, + "loss": 0.6056, + "step": 9550 + }, + { + "epoch": 0.9878073982227733, + "grad_norm": 1.4996442794799805, + "learning_rate": 1.87110393123191e-07, + "loss": 0.521, + "step": 9560 + }, + { + "epoch": 0.9888406695598264, + "grad_norm": 0.999030590057373, + "learning_rate": 1.567437388368398e-07, + "loss": 0.6173, + "step": 9570 + }, + { + "epoch": 0.9898739408968795, + "grad_norm": 2.4354379177093506, + "learning_rate": 1.2906332761647433e-07, + "loss": 0.6978, + "step": 9580 + }, + { + "epoch": 0.9909072122339326, + "grad_norm": 1.067911148071289, + "learning_rate": 1.0406945707391868e-07, + "loss": 0.5733, + "step": 9590 + }, + { + "epoch": 0.9919404835709857, + "grad_norm": 0.8200865387916565, + "learning_rate": 8.176239593610246e-08, + "loss": 0.602, + "step": 9600 + }, + { + "epoch": 0.9929737549080389, + "grad_norm": 0.7436734437942505, + "learning_rate": 6.214238404214645e-08, + "loss": 0.6507, + "step": 9610 + }, + { + "epoch": 0.994007026245092, + "grad_norm": 1.0331168174743652, + "learning_rate": 4.520963234083686e-08, + "loss": 0.6224, + "step": 9620 + }, + { + "epoch": 0.9950402975821451, + "grad_norm": 1.3714491128921509, + "learning_rate": 3.09643228882106e-08, + "loss": 0.6278, + "step": 9630 + }, + { + "epoch": 0.9960735689191982, + "grad_norm": 1.478288173675537, + "learning_rate": 1.940660884577894e-08, + "loss": 0.6508, + "step": 9640 + }, + { + "epoch": 0.9971068402562513, + "grad_norm": 1.7626447677612305, + "learning_rate": 1.053661447877885e-08, + "loss": 0.5287, + "step": 9650 + }, + { + "epoch": 0.9981401115933044, + "grad_norm": 1.4299039840698242, + "learning_rate": 4.354435154813042e-09, + "loss": 0.6035, + "step": 9660 + }, + { + "epoch": 0.9991733829303575, + "grad_norm": 0.8753514885902405, + "learning_rate": 8.601373429339976e-10, + "loss": 0.6028, + "step": 9670 + }, + { + "epoch": 1.0, + "eval_loss": 2.729660987854004, + "eval_runtime": 5409.1379, + "eval_samples_per_second": 9.159, + "eval_steps_per_second": 0.286, + "step": 9678 + }, + { + "epoch": 1.0, + "step": 9678, + "total_flos": 0.0, + "train_loss": 0.9993063533914609, + "train_runtime": 82593.2232, + "train_samples_per_second": 7.499, + "train_steps_per_second": 0.117 + } + ], + "logging_steps": 10, + "max_steps": 9678, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/__pycache__/collator.cpython-312.pyc b/__pycache__/collator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73d54aaff5f95b9639641661e9719fe30e9b448c Binary files /dev/null and b/__pycache__/collator.cpython-312.pyc differ diff --git a/__pycache__/data.cpython-312.pyc b/__pycache__/data.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc468ba1397dbc07f2cce383ae6bb9cc7342db44 Binary files /dev/null and b/__pycache__/data.cpython-312.pyc differ diff --git a/__pycache__/data_finetune.cpython-312.pyc b/__pycache__/data_finetune.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..67de8bb7a6a83fd5b789f4efbd64f2ac6eae21bc Binary files /dev/null and b/__pycache__/data_finetune.cpython-312.pyc differ diff --git a/__pycache__/evaluate.cpython-312.pyc b/__pycache__/evaluate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd9c7e9fb99f6e5b23d4353d5256c64ac8f358cf Binary files /dev/null and b/__pycache__/evaluate.cpython-312.pyc differ diff --git a/__pycache__/prompt.cpython-312.pyc b/__pycache__/prompt.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da288a4e1ef1c51869cf8e276220fe356ebd408a Binary files /dev/null and b/__pycache__/prompt.cpython-312.pyc differ diff --git a/__pycache__/prompt_finetune.cpython-312.pyc b/__pycache__/prompt_finetune.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99f7a0f0c3ed3ee76f605469952709820071a8fd Binary files /dev/null and b/__pycache__/prompt_finetune.cpython-312.pyc differ diff --git a/__pycache__/rq_llama.cpython-312.pyc b/__pycache__/rq_llama.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff48bd324b456040320a6db540ac6d9ec2887d94 Binary files /dev/null and b/__pycache__/rq_llama.cpython-312.pyc differ diff --git a/__pycache__/utils.cpython-312.pyc b/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f064a3900390c6cb8a8d37d8598824ba77fe0825 Binary files /dev/null and b/__pycache__/utils.cpython-312.pyc differ diff --git a/collator.py b/collator.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d74e2bf0c8a4181a694d20afe585ec8c51a929 --- /dev/null +++ b/collator.py @@ -0,0 +1,272 @@ +import torch +import copy +import argparse +from dataclasses import dataclass + +import transformers +import math +from torch.utils.data import Sampler +import torch.distributed as dist +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig, T5Tokenizer, T5Config, T5ForConditionalGeneration + +class VanillaCollator(object): + def __init__(self, args, tokenizer): + self.args = args + self.tokenizer = tokenizer + def __call__(self, data): + # print('collator data:',data) + ''' + [{ + 'input_ids': + "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n + ### Instruction:\n + Access the user's historical item interaction records: {inters}. + Your objective is to describe the next potential item for him, taking into account his past interactions.\n\n + ### Response:", + 'labels': + "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n + ### Instruction:\n + Access the user's historical item interaction records: {inters}. + Your objective is to describe the next potential item for him, taking into account his past interactions.\n\n + ### Response: + Dunlop guitar picks are a top choice of today's pro musician! Dunlop's wide variety of gauges, shapes, sizes and materials + allows the player to select the exact pick for his/her own particular style of playing. From classic country to nu-metal, + every great player knows that their pick is an integral part of their tone, and Dunlop guitar picks are the picks that more + pros rely on in the studio or on stage. Picks are a grossly underrated accessory. Don't sacrifice your tone...pick Dunlop guitar picks!.", + 'inters': '341,2804,3895,3893,7064', + 'item': 'placeholder', + 'task': 'inters2description' + }, + { + 'input_ids': + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n + ### Instruction:\n + Based on the user\'s historical interactions with the following items: {inters}. + You can infer his preference by observing the historical interactions: "The user\'s short-term preferences have shift to heavier picks, + suggesting that He is looking for a heavier sound.". Now the user wants a new item and searches for: "I like the durability and + effectiveness of the picks.". Please select a suitable item that matches his preference and search intent.\n\n + ### Response:', + 'labels': + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n + ### Instruction:\n + Based on the user\'s historical interactions with the following items: {inters}. + You can infer his preference by observing the historical interactions: "The user\'s short-term preferences have shift to heavier picks, + suggesting that He is looking for a heavier sound.". Now the user wants a new item and searches for: "I like the durability and + effectiveness of the picks.". Please select a suitable item that matches his preference and search intent.\n\n + ### Response:{item}', + 'inters': '122,469,8918', + 'item': '7140', + 'task': 'itemsearch' + }] + ''' + dict_data = { + 'input_ids': [], + 'labels': [], + 'inters': [], + 'item': [], + 'task': [] + } + + for d in data: + for k in dict_data.keys(): + if k == 'labels': + dict_data[k].append(d[k] + self.tokenizer.eos_token) + else: + dict_data[k].append(d[k]) + + return dict_data + +class TestCollator(object): + def __init__(self, args, tokenizer): + self.args = args + self.tokenizer = tokenizer + if self.tokenizer.pad_token_id is None: + self.tokenizer.pad_token_id = 0 + + if isinstance(self.tokenizer, LlamaTokenizer): + self.tokenizer.padding_side = "left" + + def __call__(self, batch): + input_texts = [d["input_ids"] for d in batch] + targets = [d["labels"] for d in batch] + inputs = self.tokenizer( + text = input_texts, + return_tensors ="pt", + padding = "longest", + max_length = self.tokenizer.model_max_length, + truncation = True, + return_attention_mask = True, + ) + + return (inputs, targets) + +class Collator(object): + + def __init__(self, args, tokenizer): + self.args = args + self.only_train_response = args.only_train_response + self.tokenizer = tokenizer + if self.tokenizer.pad_token_id is None: + self.tokenizer.pad_token_id = self.tokenizer.unk_token_id + # print(self.tokenizer.model_max_length) + + def __call__(self, batch): + + input_texts = [d["input_ids"] for d in batch] + full_texts = [d["labels"] + self.tokenizer.eos_token for d in batch] + + inputs = self.tokenizer( + text = full_texts, + text_target = input_texts, + return_tensors="pt", + padding="longest", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_attention_mask=True, + ) + labels = copy.deepcopy(inputs["input_ids"]) + if self.only_train_response: + # ignore padding + labels[labels == self.tokenizer.pad_token_id] = -100 + # ignore input text + labels[torch.where(inputs["labels"] != self.tokenizer.pad_token_id)] = -100 + + inputs["labels"] = labels + + + return inputs + +# RuntimeError: Cannot re-initialize CUDA in forked subprocess. +# To use CUDA with multiprocessing, you must use the 'spawn' start method. +# class ValidCollator(object): +# def __init__(self, args, model): +# self.args = args +# self.model = model +# self.only_train_response = args.only_train_response +# self.tokenizer = model.tokenizer +# def __call__(self, data): +# llama_model = self.model.model.get_decoder() +# for d in data: +# inter_emb_list = [] +# inter_item_list = d['inters'].split(',') +# for inter_item in inter_item_list: +# inter_feature = self.model.item_texts[inter_item]['title'] + ' ' + self.model.item_texts[inter_item]['description'] +# inter_id = self.tokenizer(inter_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) +# inter_emb = llama_model(input_ids = inter_id.input_ids, attention_mask = inter_id.attention_mask) +# inter_emb = inter_emb.last_hidden_state * inter_id.attention_mask.unsqueeze(-1) +# inter_emb = inter_emb.sum(dim=1) / inter_id.attention_mask.sum(dim = -1, keepdim = True) +# inter_emb_list.append(inter_emb.detach()) +# inter_embs = torch.cat(inter_emb_list, dim = 0) +# item_feature = self.model.item_texts[d['item']]['title'] + ' ' + self.model.item_texts[d['item']]['description'] +# item_ids = self.tokenizer(item_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) +# item_emb = llama_model(input_ids = item_ids.input_ids, attention_mask = item_ids.attention_mask) +# item_emb = item_emb.last_hidden_state * item_ids.attention_mask.unsqueeze(-1) +# item_emb = item_emb.sum(dim=1) / item_ids.attention_mask.sum(dim = -1, keepdim = True) +# item_emb = item_emb.detach() + +# rqids = self.model.rqvae.get_indices(torch.cat([inter_embs, item_emb], dim = 0)) + +# inters_rqids = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist()[:-1] +# item_rqid = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist()[-1] + +# text_rqids = {} +# code = '' +# for rqid in inters_rqids: +# for k, idx in enumerate(rqid): +# code = code + self.model.prefix[k].format(idx) +# code = code + ', ' +# text_rqids['inters'] = code[:-2] +# code = '' +# for k, idx in enumerate(item_rqid): +# code = code + self.model.prefix[k].format(idx) +# text_rqids['item'] = code + +# d['input_ids'] = d['input_ids'].format(inters = text_rqids['inters']) +# d['labels'] = d['labels'].format(inters = text_rqids['inters'], item = text_rqids['item']) + +# input_texts = [d["input_ids"] for d in data] +# full_texts = [d["labels"] + self.tokenizer.eos_token for d in data] + +# inputs = self.tokenizer( +# text = full_texts, +# text_target = input_texts, +# return_tensors="pt", +# padding="longest", +# max_length=self.tokenizer.model_max_length, +# truncation=True, +# return_attention_mask=True, +# ) + +# labels = copy.deepcopy(inputs["input_ids"]) +# if self.only_train_response: +# labels[labels == self.tokenizer.pad_token_id] = -100 +# labels[torch.where(inputs["labels"] != self.tokenizer.pad_token_id)] = -100 +# inputs["labels"] = labels + +# return inputs + +# RuntimeError: Cannot re-initialize CUDA in forked subprocess. +# To use CUDA with multiprocessing, you must use the 'spawn' start method. +# class TestCollator(object): +# def __init__(self, args, model): +# self.args = args +# self.model = model +# self.tokenizer = model.tokenizer +# if self.tokenizer.pad_token_id is None: +# self.tokenizer.pad_token_id = 0 +# if isinstance(self.tokenizer, LlamaTokenizer): +# self.tokenizer.padding_side = "left" + +# def __call__(self, data): +# llama_model = self.model.model.get_decoder() +# for d in data: +# inter_emb_list = [] +# inter_item_list = d['inters'].split(',') +# for inter_item in inter_item_list: +# inter_feature = self.model.item_texts[inter_item]['title'] + ' ' + self.model.item_texts[inter_item]['description'] +# inter_id = self.tokenizer(inter_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) +# inter_emb = llama_model(input_ids = inter_id.input_ids, attention_mask = inter_id.attention_mask) +# inter_emb = inter_emb.last_hidden_state * inter_id.attention_mask.unsqueeze(-1) +# inter_emb = inter_emb.sum(dim=1) / inter_id.attention_mask.sum(dim = -1, keepdim = True) +# inter_emb_list.append(inter_emb.detach()) +# inter_embs = torch.cat(inter_emb_list, dim = 0) +# item_feature = self.model.item_texts[d['item']]['title'] + ' ' + self.model.item_texts[d['item']]['description'] +# item_ids = self.tokenizer(item_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) +# item_emb = llama_model(input_ids = item_ids.input_ids, attention_mask = item_ids.attention_mask) +# item_emb = item_emb.last_hidden_state * item_ids.attention_mask.unsqueeze(-1) +# item_emb = item_emb.sum(dim=1) / item_ids.attention_mask.sum(dim = -1, keepdim = True) +# item_emb = item_emb.detach() + +# rqids = self.model.rqvae.get_indices(torch.cat([inter_embs, item_emb], dim = 0)) + +# inters_rqids = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist()[:-1] +# item_rqid = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist()[-1] + +# text_rqids = {} +# code = '' +# for rqid in inters_rqids: +# for k, idx in enumerate(rqid): +# code = code + self.model.prefix[k].format(idx) +# code = code + ', ' +# text_rqids['inters'] = code[:-2] +# code = '' +# for k, idx in enumerate(item_rqid): +# code = code + self.model.prefix[k].format(idx) +# text_rqids['item'] = code + +# d['input_ids'] = d['input_ids'].format(inters = text_rqids['inters']) +# d['labels'] = d['labels'].format(inters = text_rqids['inters'], item = text_rqids['item']) + +# input_texts = [d["input_ids"] for d in data] +# targets = [d["labels"] for d in data] + +# inputs = self.tokenizer( +# text=input_texts, +# return_tensors="pt", +# padding="longest", +# max_length=self.tokenizer.model_max_length, +# truncation=True, +# return_attention_mask=True, +# ) + +# return (inputs, targets) \ No newline at end of file diff --git a/config/ds_z2_bf16.json b/config/ds_z2_bf16.json new file mode 100644 index 0000000000000000000000000000000000000000..838e00e2b903ab73bd9fa2cc4ce436a84d944c23 --- /dev/null +++ b/config/ds_z2_bf16.json @@ -0,0 +1,28 @@ +{ + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 5e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 5e8, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false, + "flops_profiler": { + "enabled": true, + "profile_step": 10, + "module_depth": -1, + "top_modules": 3, + "detailed": true, + "output_file": "flops_profiler.out" + } +} \ No newline at end of file diff --git a/config/ds_z2_fp16.json b/config/ds_z2_fp16.json new file mode 100644 index 0000000000000000000000000000000000000000..d15699e50636bf2536592312c37162c070d586df --- /dev/null +++ b/config/ds_z2_fp16.json @@ -0,0 +1,34 @@ +{ + "fp16": { + "enabled": "auto", + "auto_cast": false, + "loss_scale": 0, + "initial_scale_power": 16, + "loss_scale_window": 1000, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 5e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 5e8, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false, + "flops_profiler": { + "enabled": true, + "profile_step": 10, + "module_depth": -1, + "top_modules": 3, + "detailed": true, + "output_file": "flops_profiler.out" + } +} \ No newline at end of file diff --git a/config/ds_z3_bf16.json b/config/ds_z3_bf16.json new file mode 100644 index 0000000000000000000000000000000000000000..0f4ca4a761aa64ec0d2953ce828c83704a1a77a4 --- /dev/null +++ b/config/ds_z3_bf16.json @@ -0,0 +1,31 @@ +{ + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": false + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false, + "flops_profiler": { + "enabled": true, + "profile_step": 10, + "module_depth": -1, + "top_modules": 3, + "detailed": true, + "output_file": "flops_profiler.out" + } +} \ No newline at end of file diff --git a/config/ds_z3_bf16_save16bit.json b/config/ds_z3_bf16_save16bit.json new file mode 100644 index 0000000000000000000000000000000000000000..fc109cbd6b6531b5cb57cab2f121e1cfda645ba4 --- /dev/null +++ b/config/ds_z3_bf16_save16bit.json @@ -0,0 +1,31 @@ +{ + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false, + "flops_profiler": { + "enabled": true, + "profile_step": 10, + "module_depth": -1, + "top_modules": 3, + "detailed": true, + "output_file": "flops_profiler.out" + } +} \ No newline at end of file diff --git a/config/ds_z3_fp16.json b/config/ds_z3_fp16.json new file mode 100644 index 0000000000000000000000000000000000000000..dbb387fccc9d72cdc3627226fb2a430e1d15418b --- /dev/null +++ b/config/ds_z3_fp16.json @@ -0,0 +1,37 @@ +{ + "fp16": { + "enabled": "auto", + "auto_cast": false, + "loss_scale": 0, + "initial_scale_power": 16, + "loss_scale_window": 1000, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": false + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false, + "flops_profiler": { + "enabled": true, + "profile_step": 10, + "module_depth": -1, + "top_modules": 3, + "detailed": true, + "output_file": "flops_profiler.out" + } +} \ No newline at end of file diff --git a/config/ds_z3_fp16_save16bit.json b/config/ds_z3_fp16_save16bit.json new file mode 100644 index 0000000000000000000000000000000000000000..00a09986aeeaec23f81ad14c00cebe660d330b62 --- /dev/null +++ b/config/ds_z3_fp16_save16bit.json @@ -0,0 +1,37 @@ +{ + "fp16": { + "enabled": "auto", + "auto_cast": false, + "loss_scale": 0, + "initial_scale_power": 16, + "loss_scale_window": 1000, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false, + "flops_profiler": { + "enabled": true, + "profile_step": 10, + "module_depth": -1, + "top_modules": 3, + "detailed": true, + "output_file": "flops_profiler.out" + } +} \ No newline at end of file diff --git a/continue_finetune.py b/continue_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..2fb982063e57b66e95a940f278a77fd1a1e5e769 --- /dev/null +++ b/continue_finetune.py @@ -0,0 +1,108 @@ +import argparse +import os +import sys +from typing import List + +import torch +import transformers +from peft import PeftModel +from peft import ( + TaskType, + LoraConfig, + get_peft_model, + get_peft_model_state_dict, + set_peft_model_state_dict, +) +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig + +from utils import * +from collator import Collator + +import argparse +from utils import * +from rq_llama import * + +parser = argparse.ArgumentParser(description = 'rqllama-finetune') +parser = parse_finetune_args(parser) +args = parser.parse_args() + +set_seed(args.seed) +ensure_dir(args.output_dir) + +device_map = "auto" +world_size = int(os.environ.get("WORLD_SIZE", 1)) +ddp = world_size != 1 +local_rank = int(os.environ.get("LOCAL_RANK") or 0) +if local_rank == 0: + print(vars(args)) + +if ddp: + device_map = {"": local_rank} + +train_data, valid_data = load_finetune_datasets(args) + +tokenizer = LlamaTokenizer.from_pretrained(args.ckpt_path) +base_model = LlamaForCausalLM.from_pretrained(args.base_model, torch_dtype=torch.float16, low_cpu_mem_usage = True, device_map = device_map) +base_model.resize_token_embeddings(len(tokenizer)) +rqllama = PeftModel.from_pretrained(base_model, args.ckpt_path, torch_dtype = torch.float16, device_map = device_map) + +if local_rank == 0: + print("token num:", len(tokenizer)) + print("data num:", len(train_data)) + +collator = Collator(args, tokenizer) + +rqllama.train() + +if local_rank == 0: + rqllama.print_trainable_parameters() + +trainer = transformers.Trainer( + model = rqllama, + train_dataset = train_data, + eval_dataset = valid_data, + args = transformers.TrainingArguments( + seed = args.seed, + per_device_train_batch_size = args.per_device_batch_size, + per_device_eval_batch_size = args.per_device_batch_size, + gradient_accumulation_steps = args.gradient_accumulation_steps, + warmup_ratio = args.warmup_ratio, + num_train_epochs = args.epochs, + learning_rate = args.learning_rate, + weight_decay = args.weight_decay, + lr_scheduler_type = args.lr_scheduler_type, + fp16 = args.fp16, + bf16 = args.bf16, + logging_steps = args.logging_step, + optim = args.optim, + gradient_checkpointing = True, + evaluation_strategy = args.save_and_eval_strategy, + save_strategy = args.save_and_eval_strategy, + eval_steps = args.save_and_eval_steps, + save_steps = args.save_and_eval_steps, + output_dir = args.output_dir, + save_total_limit = 5, + load_best_model_at_end = True, + deepspeed = args.deepspeed, + ddp_find_unused_parameters = False if ddp else None, + report_to = None, + eval_delay = 1 if args.save_and_eval_strategy=="epoch" else 2000, + dataloader_num_workers = args.dataloader_num_workers, + dataloader_prefetch_factor = args.dataloader_prefetch_factor, + remove_unused_columns = args.remove_unused_columns, + ), + tokenizer = tokenizer, + data_collator = collator, +) +rqllama.config.use_cache = False + +if torch.__version__ >= "2" and sys.platform != "win32": + rqllama = torch.compile(rqllama) + +trainer.train(resume_from_checkpoint = args.resume_from_checkpoint) + +trainer.save_state() +trainer.save_model(output_dir = args.output_dir) + +if local_rank == 0: + print('rqllama fine-tune finished.') \ No newline at end of file diff --git a/continue_pretrain.py b/continue_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..30d9e6788ea7194c5f102edcb21e46a1eb920d7f --- /dev/null +++ b/continue_pretrain.py @@ -0,0 +1,126 @@ +import os +import sys +from typing import List +import argparse + +import wandb +import torch +import transformers +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig + +from peft import ( + TaskType, + LoraConfig, + get_peft_model, + get_peft_model_state_dict, + set_peft_model_state_dict, +) + +from collator import VanillaCollator +from rq_llama import * +from utils import * + +parser = argparse.ArgumentParser(description = 'rqllama-pretrain-more') +parser = parse_global_args(parser) +parser = parse_train_args(parser) +parser = parse_dataset_args(parser) +parser = parse_rqvae_args(parser) +parser = parse_pretrain_args(parser) +args = parser.parse_args() +wandb.init(config = args, reinit = True) + +set_seed(args.seed) +ensure_dir(args.output_dir) + +device_map = "auto" +world_size = int(os.environ.get("WORLD_SIZE", 1)) +ddp = world_size != 1 +local_rank = int(os.environ.get("LOCAL_RANK") or 0) +if local_rank == 0: + print(vars(args)) +if ddp: + device_map = {"": local_rank} + +train_data, valid_data = load_datasets(args) + +rqllama = LlamaWithRQ.from_pretrained(args.ckpt_path, torch_dtype = torch.float16, low_cpu_mem_usage = True, device_map = device_map) + +for i in range(len(args.num_emb_list)): + rqllama.rqvae.rq.vq_layers[i].initted = True + +if local_rank == 0: + print("token num:", len(rqllama.tokenizer)) + print("data num:", len(train_data)) + rqllama.tokenizer.save_pretrained(args.output_dir) + rqllama.config.save_pretrained(args.output_dir) + +if args.resume_from_checkpoint: + checkpoint_name = os.path.join(args.resume_from_checkpoint, "adapter_model.bin") + args.resume_from_checkpoint = False + if os.path.exists(checkpoint_name): + if local_rank == 0: + print(f"Restarting from {checkpoint_name}") + adapters_weights = torch.load(checkpoint_name) + rqllama.model = set_peft_model_state_dict(rqllama.model, adapters_weights) + else: + if local_rank == 0: + print(f"Checkpoint {checkpoint_name} not found") + +if local_rank == 0: + rqllama.model.print_trainable_parameters() + +if not ddp and torch.cuda.device_count() > 1: + rqllama.is_parallelizable = True + rqllama.model_parallel = True + +collator = VanillaCollator(args, rqllama.tokenizer) + +trainer = transformers.Trainer( + model = rqllama, + train_dataset = train_data, + eval_dataset = valid_data, + args = transformers.TrainingArguments( + seed = args.seed, + per_device_train_batch_size = args.per_device_batch_size, + per_device_eval_batch_size = args.per_device_batch_size, + gradient_accumulation_steps = args.gradient_accumulation_steps, + warmup_ratio = args.warmup_ratio, + num_train_epochs = args.epochs, + learning_rate = args.learning_rate, + weight_decay = args.weight_decay, + lr_scheduler_type = args.lr_scheduler_type, + fp16 = args.fp16, + bf16 = args.bf16, + logging_steps = args.logging_step, + optim = args.optim, + gradient_checkpointing = True, + evaluation_strategy = args.save_and_eval_strategy, + save_strategy = args.save_and_eval_strategy, + eval_steps = args.save_and_eval_steps, + save_steps = args.save_and_eval_steps, + output_dir = args.output_dir, + save_total_limit = 5, + load_best_model_at_end = True, + deepspeed = args.deepspeed, + ddp_find_unused_parameters = False if ddp else None, + report_to = None, + eval_delay = 1 if args.save_and_eval_strategy=="epoch" else 2000, + dataloader_num_workers = args.dataloader_num_workers, + dataloader_prefetch_factor = args.dataloader_prefetch_factor, + remove_unused_columns = args.remove_unused_columns, + ), + tokenizer = rqllama.tokenizer, + data_collator = collator, +) +rqllama.config.use_cache = False + +if torch.__version__ >= "2" and sys.platform != "win32": + rqllama = torch.compile(rqllama) + +trainer.train(resume_from_checkpoint = args.resume_from_checkpoint) + +trainer.save_state() +trainer.save_model(output_dir = args.output_dir) + +if local_rank == 0: + print('rqllama pre-train finished.') \ No newline at end of file diff --git a/convert/convert.log b/convert/convert.log new file mode 100644 index 0000000000000000000000000000000000000000..4afb9670e20cbc6fbefe1247b4f425ca9c7dc0d9 --- /dev/null +++ b/convert/convert.log @@ -0,0 +1 @@ +nohup: failed to run command './convert.sh': Permission denied diff --git a/convert/convert.py b/convert/convert.py new file mode 100644 index 0000000000000000000000000000000000000000..f23f257f0c26116db8a0480d84f9d0ac5207f068 --- /dev/null +++ b/convert/convert.py @@ -0,0 +1,16 @@ +import transformers +import argparse +import os + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--source", "-s", type=str, default="", help="source path of models") + parser.add_argument("--target", "-t", type=str, default="", help="target path of models") + + args, _ = parser.parse_known_args() + + assert os.path.exists(args.source) + assert args.target != "" + + model = transformers.AutoModelForCausalLM.from_pretrained(args.source) + model.save_pretrained(args.target, state_dict=model.state_dict()) \ No newline at end of file diff --git a/convert/convert.sh b/convert/convert.sh new file mode 100644 index 0000000000000000000000000000000000000000..da33cc755b1a61b8d811119fe969f93c97535d4d --- /dev/null +++ b/convert/convert.sh @@ -0,0 +1,18 @@ +model=$1 + +set -x + +for step in `ls ${model} | grep checkpoint | awk -F'-' '{ print $2 }'` +do +mkdir ${model}/tmp-checkpoint-${step} +mkdir ${model}/final-checkpoint-${step} +python ./zero_to_fp32.py ${model}/checkpoint-${step}/ ${model}/tmp-checkpoint-${step}/pytorch_model.bin +cp ${model}/*.json ${model}/tmp-checkpoint-${step} +python ./convert.py -s ${model}/tmp-checkpoint-${step} -t ${model}/final-checkpoint-${step} +cp ${model}/checkpoint-${step}/*.json ${model}/final-checkpoint-${step} +cp ${model}/*.json ${model}/final-checkpoint-${step} +cp ${model}/tokenizer* ${model}/final-checkpoint-${step} +cp ${model}/train* ${model}/final-checkpoint-${step} +#rm -rf ${model}/tmp-checkpoint-${step} ${model}/checkpoint-${step} ${model}/global_step${step} +#mv ${model}/final-checkpoint-${step} ${model}/checkpoint-${step} +done \ No newline at end of file diff --git a/convert/convert_fp16.py b/convert/convert_fp16.py new file mode 100644 index 0000000000000000000000000000000000000000..e23216d65b1b937d1c823e030df0ab569aec989a --- /dev/null +++ b/convert/convert_fp16.py @@ -0,0 +1,23 @@ + +import argparse + +from transformers import AutoTokenizer, AutoModelForCausalLM +import torch + + +def convert_fp16(in_checkpoint, out_checkpoint): + tokenizer = AutoTokenizer.from_pretrained(in_checkpoint, use_fast=False) + model = AutoModelForCausalLM.from_pretrained( + in_checkpoint, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + model.save_pretrained(out_checkpoint) + tokenizer.save_pretrained(out_checkpoint) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--in-checkpoint", type=str, help="Path to the model") + parser.add_argument("--out-checkpoint", type=str, help="Path to the output model") + args = parser.parse_args() + + convert_fp16(args.in_checkpoint, args.out_checkpoint) diff --git a/convert/make_delta.py b/convert/make_delta.py new file mode 100644 index 0000000000000000000000000000000000000000..69fcf88b267e60fc208f97964324a4e41b39d239 --- /dev/null +++ b/convert/make_delta.py @@ -0,0 +1,46 @@ + +import argparse + +import torch +from tqdm import tqdm +from transformers import AutoTokenizer, AutoModelForCausalLM + + +def make_delta(base_model_path, target_model_path, delta_path): + print(f"Loading the base model from {base_model_path}") + base = AutoModelForCausalLM.from_pretrained( + base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + + print(f"Loading the target model from {target_model_path}") + target = AutoModelForCausalLM.from_pretrained( + target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + target_tokenizer = AutoTokenizer.from_pretrained(target_model_path, use_fast=False) + + print("Calculating the delta") + for name, param in tqdm(target.state_dict().items(), desc="Calculating delta"): + assert name in base.state_dict() + if param.shape == base.state_dict()[name].shape: + param.data -= base.state_dict()[name] + else: + print(name) + + print(f"Saving the delta to {delta_path}") + if args.hub_repo_id: + kwargs = {"push_to_hub": True, "repo_id": args.hub_repo_id} + else: + kwargs = {} + target.save_pretrained(delta_path, **kwargs) + target_tokenizer.save_pretrained(delta_path, **kwargs) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--base-model-path", type=str, required=True) + parser.add_argument("--target-model-path", type=str, required=True) + parser.add_argument("--delta-path", type=str, required=True) + parser.add_argument("--hub-repo-id", type=str) + args = parser.parse_args() + + make_delta(args.base_model_path, args.target_model_path, args.delta_path) diff --git a/convert/merge_delta.py b/convert/merge_delta.py new file mode 100644 index 0000000000000000000000000000000000000000..dabbe3a3c77ee75e0ec382caa38c64289febef0e --- /dev/null +++ b/convert/merge_delta.py @@ -0,0 +1,167 @@ + +import argparse +import gc +import glob +import json +import os +import shutil +import tempfile + +from huggingface_hub import snapshot_download +import torch +from torch import nn +from tqdm import tqdm +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig + + +GB = 1 << 30 + + +def split_files(model_path, tmp_path, split_size): + if not os.path.exists(model_path): + model_path = snapshot_download(repo_id=model_path) + if not os.path.exists(tmp_path): + os.makedirs(tmp_path) + + file_pattern = os.path.join(model_path, "pytorch_model-*.bin") + files = glob.glob(file_pattern) + + part = 0 + try: + for file_path in tqdm(files): + state_dict = torch.load(file_path) + new_state_dict = {} + + current_size = 0 + for name, param in state_dict.items(): + param_size = param.numel() * param.element_size() + + if current_size + param_size > split_size: + new_file_name = f"pytorch_model-{part}.bin" + new_file_path = os.path.join(tmp_path, new_file_name) + torch.save(new_state_dict, new_file_path) + current_size = 0 + new_state_dict = None + gc.collect() + new_state_dict = {} + part += 1 + + new_state_dict[name] = param + current_size += param_size + + new_file_name = f"pytorch_model-{part}.bin" + new_file_path = os.path.join(tmp_path, new_file_name) + torch.save(new_state_dict, new_file_path) + new_state_dict = None + gc.collect() + new_state_dict = {} + part += 1 + except Exception as e: + print(f"An error occurred during split_files: {e}") + shutil.rmtree(tmp_path) + raise + + +def apply_delta_low_cpu_mem(base_model_path, target_model_path, delta_path): + delta_tokenizer = AutoTokenizer.from_pretrained(delta_path, use_fast=False) + delta_config = AutoConfig.from_pretrained(delta_path) + + if os.path.exists(target_model_path): + shutil.rmtree(target_model_path) + os.makedirs(target_model_path) + + split_size = 4 * GB + + with tempfile.TemporaryDirectory() as tmp_base_path, tempfile.TemporaryDirectory() as tmp_delta_path: + print(f"Split files for the base model to {tmp_base_path}") + split_files(base_model_path, tmp_base_path, split_size) + print(f"Split files for the delta weights to {tmp_delta_path}") + split_files(delta_path, tmp_delta_path, split_size) + + base_pattern = os.path.join(tmp_base_path, "pytorch_model-*.bin") + base_files = glob.glob(base_pattern) + base_state_dict = torch.load(base_files[0]) + delta_pattern = os.path.join(tmp_delta_path, "pytorch_model-*.bin") + delta_files = glob.glob(delta_pattern) + # delta_state_dict = torch.load(delta_files[0]) + + print("Applying the delta") + weight_map = {} + total_size = 0 + + for i, delta_file in tqdm(enumerate(delta_files)): + state_dict = torch.load(delta_file) + file_name = f"pytorch_model-{i}.bin" + for name, param in state_dict.items(): + if name not in base_state_dict: + for base_file in base_files: + base_state_dict = torch.load(base_file) + gc.collect() + if name in base_state_dict: + break + if state_dict[name].shape == base_state_dict[name].shape: + state_dict[name] += base_state_dict[name] + else: + print(name) + weight_map[name] = file_name + total_size += param.numel() * param.element_size() + gc.collect() + torch.save(state_dict, os.path.join(target_model_path, file_name)) + + with open( + os.path.join(target_model_path, "pytorch_model.bin.index.json"), "w" + ) as f: + json.dump( + {"weight_map": weight_map, "metadata": {"total_size": total_size}}, f + ) + + print(f"Saving the target model to {target_model_path}") + delta_tokenizer.save_pretrained(target_model_path) + delta_config.save_pretrained(target_model_path) + + +def apply_delta(base_model_path, target_model_path, delta_path): + print(f"Loading the delta weights from {delta_path}") + delta_tokenizer = AutoTokenizer.from_pretrained(delta_path, use_fast=False) + delta = AutoModelForCausalLM.from_pretrained( + delta_path, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + + print(f"Loading the base model from {base_model_path}") + base = AutoModelForCausalLM.from_pretrained( + base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + + print("Applying the delta") + for name, param in tqdm(delta.state_dict().items(), desc="Applying delta"): + assert name in base.state_dict() + if param.shape == base.state_dict()[name].shape: + param.data += base.state_dict()[name] + else: + print(name) + + + print(f"Saving the target model to {target_model_path}") + delta.save_pretrained(target_model_path) + delta_tokenizer.save_pretrained(target_model_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--base-model-path", type=str, required=True) + parser.add_argument("--target-model-path", type=str, required=True) + parser.add_argument("--delta-path", type=str, required=True) + parser.add_argument( + "--low-cpu-mem", + action="store_true", + help="Lower the cpu memory usage. This will split large files and use " + "disk as swap to reduce the memory usage below 10GB.", + ) + args = parser.parse_args() + + if args.low_cpu_mem: + apply_delta_low_cpu_mem( + args.base_model_path, args.target_model_path, args.delta_path + ) + else: + apply_delta(args.base_model_path, args.target_model_path, args.delta_path) diff --git a/convert/zero_to_fp32.py b/convert/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..12f6efbbd4e8530ef997c08d95a6a4460039c3ce --- /dev/null +++ b/convert/zero_to_fp32.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass +from tqdm import tqdm + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage == 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # record shared parameters so that they can be recovered based on partners + # this is because such parameters holding reference only are not saved by optimizer + shared_params = [] + for param in state_dict["module"]: + if param not in [*param_names, *buffer_names]: + for share_param in state_dict["module"]: + if (state_dict["module"][share_param].data_ptr() == state_dict["module"][param].data_ptr() + and share_param != param): + shared_params.append([param, share_param]) + break + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for i, f in enumerate(tqdm(files)): + state_dicts.append(torch.load(f, map_location=device)) + if i == 0: + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage == 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + key_list = list(state_dicts[-1][OPTIMIZER_STATE_DICT].keys()) + for key in key_list: + if zero_stage == 2: + if key != fp32_groups_key: + del state_dicts[-1][OPTIMIZER_STATE_DICT][key] + elif zero_stage == 3: + if key == fp32_groups_key: + value = torch.cat(state_dicts[-1][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) + del state_dicts[-1][OPTIMIZER_STATE_DICT][key] + if key == fp32_groups_key: + state_dicts[-1][OPTIMIZER_STATE_DICT][key] = value + + print('zero_stage:', zero_stage) + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + # if zero_stage == 2: + # # fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + # elif zero_stage == 3: + # # if there is more than one param group, there will be multiple flattened tensors - one + # # flattened tensor per group - for simplicity merge them into a single tensor + # # + # # XXX: could make the script more memory efficient for when there are multiple groups - it + # # will require matching the sub-lists of param_shapes for each param group flattened tensor + + # print('start!') + # # fp32_flat_groups = [ + # # torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + # # ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage == 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in tqdm(zero_model_states[0].frozen_param_shapes.items()): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items()): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file) diff --git a/data_finetune.py b/data_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..265543b07e977e44884739bd8ac2e3ac0ad4d308 --- /dev/null +++ b/data_finetune.py @@ -0,0 +1,852 @@ +import copy +import random +import argparse +import os +import torch +import torch.nn as nn +from torch.utils.data import Dataset +from tqdm import tqdm +from collections import defaultdict +import torch.distributed as dist +import logging +import re +import pdb +import json +from prompt_finetune import sft_prompt, all_prompt +import numpy as np + + +class BaseDataset(Dataset): + + def __init__(self, args): + super().__init__() + + self.args = args + self.dataset = args.dataset + self.data_path = os.path.join(args.data_path, self.dataset) + + self.max_his_len = args.max_his_len + self.his_sep = args.his_sep + self.index_file = args.index_file + self.add_prefix = args.add_prefix + + self.new_tokens = None + self.allowed_tokens = None + self.all_items = None + + + def _load_data(self): + + with open(os.path.join(self.data_path, self.dataset + self.index_file), 'r') as f: + self.indices = json.load(f) + + def get_new_tokens(self): + + if self.new_tokens is not None: + return self.new_tokens + + self.new_tokens = set() + for index in self.indices.values(): + for token in index: + self.new_tokens.add(token) + self.new_tokens = sorted(list(self.new_tokens)) + + return self.new_tokens + + def get_all_items(self): + + if self.all_items is not None: + return self.all_items + + self.all_items = set() + for index in self.indices.values(): + self.all_items.add("".join(index)) + + return self.all_items + + def get_prefix_allowed_tokens_fn(self, tokenizer): + + + if self.allowed_tokens is None: + self.allowed_tokens = {} + for index in self.indices.values(): + for i, token in enumerate(index): + token_id = tokenizer(token)["input_ids"][1] + if i not in self.allowed_tokens.keys(): + self.allowed_tokens[i] = set() + self.allowed_tokens[i].add(token_id) + self.allowed_tokens[len(self.allowed_tokens.keys())] = set([tokenizer.eos_token_id]) + sep = tokenizer("Response:")["input_ids"][1:] + + def prefix_allowed_tokens_fn(batch_id, sentence): + sentence = sentence.tolist() + reversed_sent = sentence[::-1] + for i in range(len(reversed_sent)): + if reversed_sent[i:i + len(sep)] == sep[::-1]: + # print(list(self.allowed_tokens[i])) + return list(self.allowed_tokens[i]) + + return prefix_allowed_tokens_fn + + def _process_data(self): + + raise NotImplementedError + + + +class SeqRecFinetune(BaseDataset): + + def __init__(self, args, mode="train", + prompt_sample_num=1, prompt_id=0, sample_num=-1): + super().__init__(args) + + self.mode = mode + self.prompt_sample_num = prompt_sample_num + self.prompt_id = prompt_id + self.sample_num = sample_num + + self.prompts = all_prompt["seqrec"] + + + # load data + self._load_data() + self._remap_items() + + # load data + if self.mode == 'train': + self.inter_data = self._process_train_data() + elif self.mode == 'valid': + self.sample_valid = args.sample_valid + self.valid_prompt_id = args.valid_prompt_id + self.inter_data = self._process_valid_data() + self._construct_valid_text() + elif self.mode == 'test': + self.inter_data = self._process_test_data() + else: + raise NotImplementedError + + + + def _load_data(self): + + with open(os.path.join(self.data_path, self.dataset + ".inter.json"), 'r') as f: + self.inters = json.load(f) + with open(self.index_file, 'r') as f: + self.indices = json.load(f) + + + def _remap_items(self): + + self.remapped_inters = dict() + for uid, items in self.inters.items(): + new_items = ["".join(self.indices[str(i)]) for i in items] + self.remapped_inters[uid] = new_items + + + def _process_train_data(self): + + inter_data = [] + for uid in self.remapped_inters: + items = self.remapped_inters[uid][:-2] + for i in range(1, len(items)): + one_data = dict() + # one_data["user"] = uid + one_data["item"] = items[i] + history = items[:i] + if self.max_his_len > 0: + history = history[-self.max_his_len:] + if self.add_prefix: + history = [str(k+1) + ". " + item_idx for k, item_idx in enumerate(history)] + one_data["inters"] = self.his_sep.join(history) + inter_data.append(one_data) + + return inter_data + + def _process_valid_data(self): + + inter_data = [] + for uid in self.remapped_inters: + items = self.remapped_inters[uid] + one_data = dict() + # one_data["user"] = uid + one_data["item"] = items[-2] + history = items[:-2] + if self.max_his_len > 0: + history = history[-self.max_his_len:] + if self.add_prefix: + history = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(history)] + one_data["inters"] = self.his_sep.join(history) + inter_data.append(one_data) + + return inter_data + + def _process_test_data(self): + + inter_data = [] + for uid in self.remapped_inters: + items = self.remapped_inters[uid] + one_data = dict() + # one_data["user"] = uid + one_data["item"] = items[-1] + history = items[:-1] + if self.max_his_len > 0: + history = history[-self.max_his_len:] + if self.add_prefix: + history = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(history)] + one_data["inters"] = self.his_sep.join(history) + inter_data.append(one_data) + + if self.sample_num > 0: + all_inter_idx = range(len(inter_data)) + sample_idx = np.random.choice(all_inter_idx, self.sample_num, replace=False) + inter_data = np.array(inter_data)[sample_idx].tolist() + + return inter_data + + def set_prompt(self, prompt_id): + + self.prompt_id = prompt_id + + def __len__(self): + if self.mode == 'train': + return len(self.inter_data) * self.prompt_sample_num + elif self.mode == 'valid': + return len(self.valid_text_data) + elif self.mode == 'test': + return len(self.inter_data) + else: + raise NotImplementedError + + def _construct_valid_text(self): + self.valid_text_data = [] + if self.sample_valid: + all_prompt_ids = range(len(self.prompts)) + for i in range(len(self.inter_data)): + d = self.inter_data[i] + prompt_ids = np.random.choice(all_prompt_ids, self.prompt_sample_num, replace=False) + for prompt_id in prompt_ids: + prompt = self.prompts[prompt_id] + input, output = self._get_text_data(d, prompt) + self.valid_text_data.append({"input_ids": input, "labels": output}) + else: + self.prompt_sample_num = 1 + prompt = self.prompts[self.valid_prompt_id] + for i in range(len(self.inter_data)): + d = self.inter_data[i] + input, output = self._get_text_data(d, prompt) + self.valid_text_data.append({"input_ids": input, "labels": output}) + + def _get_text_data(self, data, prompt): + + instruction = prompt["instruction"].format(**data) + response = prompt["response"].format(**data) + + input = sft_prompt.format(instruction = instruction, response = "") + output = sft_prompt.format(instruction = instruction, response = response) + + if self.mode == 'test': + return input, response + + return input, output + + def __getitem__(self, index): + + if self.mode == 'valid': + return self.valid_text_data[index] + + idx = index // self.prompt_sample_num + d = self.inter_data[idx] + # print(index, idx) + + if self.mode == 'train': + prompt_id = random.randint(0, len(self.prompts) - 1) + elif self.mode == 'test': + prompt_id = self.prompt_id + + prompt = self.prompts[prompt_id] + + input, output = self._get_text_data(d, prompt) + + # print({"input": input, "output": output}) + + return dict(input_ids=input, labels=output) + + +class FusionSeqRecFinetune(BaseDataset): + + def __init__(self, args, mode="train", + prompt_sample_num=1, prompt_id=0, sample_num=-1): + super().__init__(args) + + self.mode = mode + self.prompt_sample_num = prompt_sample_num + self.prompt_id = prompt_id + self.sample_num = sample_num + + self.prompts = all_prompt["fusionseqrec"] + + # load data + self._load_data() + # self._remap_items() + + # load data + if self.mode == 'train': + self.inter_data = self._process_train_data() + elif self.mode == 'valid': + self.sample_valid = args.sample_valid + self.valid_prompt_id = args.valid_prompt_id + self.inter_data = self._process_valid_data() + self._construct_valid_text() + elif self.mode == 'test': + self.inter_data = self._process_test_data() + else: + raise NotImplementedError + + + def _load_data(self): + + with open(os.path.join(self.data_path, self.dataset + ".inter.json"), 'r') as f: + self.inters = json.load(f) + with open(self.index_file, 'r') as f: + self.indices = json.load(f) + # with open(os.path.join(self.data_path, self.dataset + self.index_file), 'r') as f: + # self.indices = json.load(f) + with open(os.path.join(self.data_path, self.dataset + ".item.json"), 'r') as f: + self.item_feat = json.load(f) + + def _process_train_data(self): + + inter_data = [] + for uid in self.inters: + items = self.inters[uid][:-2] + for i in range(1, len(items)): + one_data = dict() + # one_data["user"] = uid + one_data["item"] = "".join(self.indices[str(items[i])]) + one_data["title"] = self.item_feat[str(items[i])]["title"].strip().strip(".!?,;:`") + one_data["description"] = self.item_feat[str(items[i])]["description"] + history = items[:i] + if self.max_his_len > 0: + history = history[-self.max_his_len:] + inters = ["".join(self.indices[str(j)]) for j in history] + inter_titles = ["\"" + self.item_feat[str(j)]["title"].strip().strip(".!?,;:`") + "\"" for j in history] + + + if self.add_prefix: + inters = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(inters)] + inter_titles = [str(k + 1) + ". " + item_title for k, item_title in enumerate(inter_titles)] + + one_data["inters"] = self.his_sep.join(inters) + one_data["inter_titles"] = self.his_sep.join(inter_titles) + inter_data.append(one_data) + + if self.sample_num > 0: + all_inter_idx = range(len(inter_data)) + sample_idx = np.random.choice(all_inter_idx, self.sample_num, replace=False) + inter_data = np.array(inter_data)[sample_idx].tolist() + + return inter_data + + def _process_valid_data(self): + + inter_data = [] + for uid in self.inters: + items = self.inters[uid] + one_data = dict() + one_data["item"] = "".join(self.indices[str(items[-2])]) + one_data["title"] = self.item_feat[str(items[-2])]["title"].strip().strip(".!?,;:`") + one_data["description"] = self.item_feat[str(items[-2])]["description"] + + + history = items[:-2] + if self.max_his_len > 0: + history = history[-self.max_his_len:] + inters = ["".join(self.indices[str(j)]) for j in history] + inter_titles = ["\"" + self.item_feat[str(j)]["title"].strip().strip(".!?,;:`") + "\"" for j in history] + + if self.add_prefix: + inters = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(inters)] + inter_titles = [str(k + 1) + ". " + item_title for k, item_title in enumerate(inter_titles)] + + one_data["inters"] = self.his_sep.join(inters) + one_data["inter_titles"] = self.his_sep.join(inter_titles) + inter_data.append(one_data) + + if self.sample_num > 0: + all_inter_idx = range(len(inter_data)) + sample_idx = np.random.choice(all_inter_idx, self.sample_num, replace=False) + inter_data = np.array(inter_data)[sample_idx].tolist() + + return inter_data + + def _process_test_data(self): + + inter_data = [] + for uid in self.inters: + items = self.inters[uid] + one_data = dict() + one_data["item"] = "".join(self.indices[str(items[-1])]) + one_data["title"] = self.item_feat[str(items[-1])]["title"].strip().strip(".!?,;:`") + one_data["description"] = self.item_feat[str(items[-1])]["description"] + + history = items[:-1] + if self.max_his_len > 0: + history = history[-self.max_his_len:] + inters = ["".join(self.indices[str(j)]) for j in history] + inter_titles = ["\"" + self.item_feat[str(j)]["title"].strip().strip(".!?,;:`") + "\"" for j in history] + + if self.add_prefix: + inters = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(inters)] + inter_titles = [str(k + 1) + ". " + item_title for k, item_title in enumerate(inter_titles)] + + one_data["inters"] = self.his_sep.join(inters) + one_data["inter_titles"] = self.his_sep.join(inter_titles) + inter_data.append(one_data) + + if self.sample_num > 0: + all_inter_idx = range(len(inter_data)) + sample_idx = np.random.choice(all_inter_idx, self.sample_num, replace=False) + inter_data = np.array(inter_data)[sample_idx].tolist() + + return inter_data + + def set_prompt(self, prompt_id): + + self.prompt_id = prompt_id + + def __len__(self): + if self.mode == 'train': + return len(self.inter_data) * self.prompt_sample_num + elif self.mode == 'valid': + return len(self.valid_text_data) + elif self.mode == 'test': + return len(self.inter_data) + else: + raise NotImplementedError + + def _construct_valid_text(self): + self.valid_text_data = [] + if self.sample_valid: + all_prompt_ids = range(len(self.prompts)) + for i in range(len(self.inter_data)): + d = self.inter_data[i] + prompt_ids = np.random.choice(all_prompt_ids, self.prompt_sample_num, replace=False) + for prompt_id in prompt_ids: + prompt = self.prompts[prompt_id] + input, output = self._get_text_data(d, prompt) + self.valid_text_data.append({"input_ids": input, "labels": output}) + else: + self.prompt_sample_num = 1 + prompt = self.prompts[self.valid_prompt_id] + for i in range(len(self.inter_data)): + d = self.inter_data[i] + input, output = self._get_text_data(d, prompt) + self.valid_text_data.append({"input_ids": input, "labels": output}) + + def _get_text_data(self, data, prompt): + + instruction = prompt["instruction"].format(**data) + response = prompt["response"].format(**data) + + input = sft_prompt.format(instruction=instruction, response="") + output = sft_prompt.format(instruction=instruction, response=response) + + if self.mode == 'test': + return input, response + + return input, output + + def __getitem__(self, index): + + if self.mode == 'valid': + return self.valid_text_data[index] + + idx = index // self.prompt_sample_num + d = self.inter_data[idx] + + if self.mode == 'train': + prompt_id = random.randint(0, len(self.prompts) - 1) + elif self.mode == 'test': + prompt_id = self.prompt_id + + prompt = self.prompts[prompt_id] + + input, output = self._get_text_data(d, prompt) + + + return dict(input_ids=input, labels=output) + + +class ItemFeatFinetune(BaseDataset): + + def __init__(self, args, task="item2index", prompt_sample_num=1, sample_num=-1): + super().__init__(args) + + self.task = task.lower() + self.prompt_sample_num = prompt_sample_num + self.sample_num = sample_num + + self.prompts = all_prompt[self.task] + + # load data + self._load_data() + self.feat_data = self._process_data() + + + + def _load_data(self): + + # with open(os.path.join(self.data_path, self.dataset + self.index_file), 'r') as f: + # self.indices = json.load(f) + with open(self.index_file, 'r') as f: + self.indices = json.load(f) + with open(os.path.join(self.data_path, self.dataset + ".item.json"), 'r') as f: + self.item_feat = json.load(f) + + + def _process_data(self): + + feat_data = [] + for iid in self.item_feat: + feat = self.item_feat[iid] + index = "".join(self.indices[iid]) + feat["item"] = index + feat["title"] = feat["title"].strip().strip(".!?,;:`") + feat_data.append(feat) + + if self.sample_num > 0: + all_idx = range(len(feat_data)) + sample_idx = np.random.choice(all_idx, self.sample_num, replace=False) + + feat_data = np.array(feat_data)[sample_idx].tolist() + + return feat_data + + + def __len__(self): + return len(self.feat_data) * self.prompt_sample_num + + def _get_text_data(self, data, prompt): + + instruction = prompt["instruction"].format(**data) + response = prompt["response"].format(**data) + + input = sft_prompt.format(instruction = instruction, response = "") + output = sft_prompt.format(instruction = instruction, response = response) + + return input, output + + def __getitem__(self, index): + + idx = index // self.prompt_sample_num + d = self.feat_data[idx] + + prompt_id = random.randint(0, len(self.prompts) - 1) + + prompt = self.prompts[prompt_id] + + input, output = self._get_text_data(d, prompt) + + return dict(input_ids=input, labels=output) + + +class ItemSearchFinetune(BaseDataset): + + def __init__(self, args, mode="train", + prompt_sample_num=1, prompt_id=0, sample_num=-1): + super().__init__(args) + + self.mode = mode + self.prompt_sample_num = prompt_sample_num + self.prompt_id = prompt_id + self.sample_num = sample_num + + self.prompts = all_prompt["itemsearch"] + + # load data + self._load_data() + self.search_data = self._process_data() + + + + def _load_data(self): + + # with open(os.path.join(self.data_path, self.dataset + self.index_file), 'r') as f: + # self.indices = json.load(f) + with open(self.index_file, 'r') as f: + self.indices = json.load(f) + with open(os.path.join(self.data_path, self.dataset + ".user.json"), 'r') as f: + self.user_info = json.load(f) + + + def _process_data(self): + + search_data = [] + user_explicit_preference = self.user_info["user_explicit_preference"] + user_vague_intention = self.user_info["user_vague_intention"] + if self.mode == 'train': + user_vague_intention = user_vague_intention["train"] + elif self.mode == 'test': + user_vague_intention = user_vague_intention["test"] + else: + raise NotImplementedError + + for uid in user_explicit_preference.keys(): + one_data = {} + user_ep = user_explicit_preference[uid] + user_vi = user_vague_intention[uid]["querys"] + one_data["explicit_preferences"] = user_ep + one_data["user_related_intention"] = user_vi[0] + one_data["item_related_intention"] = user_vi[1] + + iid = user_vague_intention[uid]["item"] + inters = user_vague_intention[uid]["inters"] + + index = "".join(self.indices[str(iid)]) + one_data["item"] = index + + if self.max_his_len > 0: + inters = inters[-self.max_his_len:] + inters = ["".join(self.indices[str(i)]) for i in inters] + if self.add_prefix: + inters = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(inters)] + + one_data["inters"] = self.his_sep.join(inters) + + search_data.append(one_data) + + if self.sample_num > 0: + all_idx = range(len(search_data)) + sample_idx = np.random.choice(all_idx, self.sample_num, replace=False) + + search_data = np.array(search_data)[sample_idx].tolist() + + return search_data + + def set_prompt(self, prompt_id): + self.prompt_id = prompt_id + + def __len__(self): + if self.mode == 'train': + return len(self.search_data) * self.prompt_sample_num + elif self.mode == 'test': + return len(self.search_data) + else: + return len(self.search_data) + + + def _get_text_data(self, data, prompt): + + instruction = prompt["instruction"].format(**data) + response = prompt["response"].format(**data) + + input = sft_prompt.format(instruction = instruction, response = "") + output = sft_prompt.format(instruction = instruction, response = response) + + if self.mode == 'test': + return input, response + + return input, output + + def __getitem__(self, index): + + idx = index // self.prompt_sample_num + + d = self.search_data[idx] + if self.mode == 'train': + prompt_id = random.randint(0, len(self.prompts) - 1) + elif self.mode == 'test': + prompt_id = self.prompt_id + + prompt = self.prompts[prompt_id] + + d["explicit_preference"] = copy.deepcopy(random.choice(d["explicit_preferences"])) + all_querys = [d["user_related_intention"], d["item_related_intention"]] + d["query"] = random.choice(all_querys) + + input, output = self._get_text_data(d, prompt) + + return dict(input_ids=input, labels=output) + + + +class PreferenceObtainFinetune(BaseDataset): + + def __init__(self, args, prompt_sample_num=1, sample_num=-1): + super().__init__(args) + + self.prompt_sample_num = prompt_sample_num + self.sample_num = sample_num + + self.prompts = all_prompt["preferenceobtain"] + + # load data + self._load_data() + self._remap_items() + + self.preference_data = self._process_data() + + + + def _load_data(self): + + with open(os.path.join(self.data_path, self.dataset + ".user.json"), 'r') as f: + self.user_info = json.load(f) + with open(os.path.join(self.data_path, self.dataset + ".inter.json"), 'r') as f: + self.inters = json.load(f) + # with open(os.path.join(self.data_path, self.dataset + self.index_file), 'r') as f: + # self.indices = json.load(f) + with open(self.index_file, 'r') as f: + self.indices = json.load(f) + + + def _remap_items(self): + + self.remapped_inters = dict() + for uid, items in self.inters.items(): + new_items = ["".join(self.indices[str(i)]) for i in items] + self.remapped_inters[uid] = new_items + + def _process_data(self): + + preference_data = [] + user_explicit_preference = self.user_info["user_explicit_preference"] + + for uid in user_explicit_preference.keys(): + one_data = {} + inters = self.remapped_inters[uid][:-3] + user_ep = user_explicit_preference[uid] + + if self.max_his_len > 0: + inters = inters[-self.max_his_len:] + if self.add_prefix: + inters = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(inters)] + + one_data["explicit_preferences"] = user_ep + one_data["inters"] = self.his_sep.join(inters) + + preference_data.append(one_data) + + if self.sample_num > 0: + all_idx = range(len(preference_data)) + sample_idx = np.random.choice(all_idx, self.sample_num, replace=False) + + preference_data = np.array(preference_data)[sample_idx].tolist() + + return preference_data + + def set_prompt(self, prompt_id): + self.prompt_id = prompt_id + + def __len__(self): + return len(self.preference_data) * self.prompt_sample_num + + + def _get_text_data(self, data, prompt): + + instruction = prompt["instruction"].format(**data) + response = prompt["response"].format(**data) + + input = sft_prompt.format(instruction = instruction, response = "") + output = sft_prompt.format(instruction = instruction, response = response) + + return input, output + + def __getitem__(self, index): + + idx = index // self.prompt_sample_num + + d = self.preference_data[idx] + prompt_id = random.randint(0, len(self.prompts) - 1) + + prompt = self.prompts[prompt_id] + + d["explicit_preference"] = copy.deepcopy(random.choice(d["explicit_preferences"])) + + input, output = self._get_text_data(d, prompt) + + return dict(input_ids=input, labels=output) + + + + + +class SeqRecTestDataset(BaseDataset): + + def __init__(self, args, prompt_id=0, sample_num=-1): + super().__init__(args) + + self.prompt_id = prompt_id + self.sample_num = sample_num + + self.prompt = all_prompt["seqrec"][self.prompt_id] + + # load data + self._load_data() + self._remap_items() + + self.inter_data = self._process_test_data() + + def _load_data(self): + + with open(os.path.join(self.data_path, self.dataset + ".inter.json"), 'r') as f: + self.inters = json.load(f) + with open(os.path.join(self.data_path, self.dataset + self.index_file), 'r') as f: + self.indices = json.load(f) + + + def _remap_items(self): + + self.remapped_inters = dict() + for uid, items in self.inters.items(): + new_items = ["".join(self.indices[str(i)]) for i in items] + self.remapped_inters[uid] = new_items + + def _process_test_data(self): + + inter_data = [] + for uid in self.remapped_inters: + items = self.remapped_inters[uid] + one_data = dict() + # one_data["user"] = uid + one_data["item"] = items[-1] + history = items[:-1] + if self.max_his_len > 0: + history = history[-self.max_his_len:] + if self.add_prefix: + history = [str(k + 1) + ". " + item_idx for k, item_idx in enumerate(history)] + one_data["inters"] = self.his_sep.join(history) + inter_data.append(one_data) + + if self.sample_num > 0: + all_inter_idx = range(len(inter_data)) + sample_idx = np.random.choice(all_inter_idx, self.sample_num, replace=False) + + inter_data = np.array(inter_data)[sample_idx].tolist() + + return inter_data + + def set_prompt(self, prompt_id): + self.prompt_id = prompt_id + + self.prompt = all_prompt["seqrec"][self.prompt_id] + + def __len__(self): + + return len(self.inter_data) + + def _get_text_data(self, data, prompt): + + instruction = prompt["instruction"].format(**data) + response = prompt["response"].format(**data) + + input = sft_prompt.format(instruction=instruction, response="") + + return input, response + + def __getitem__(self, index): + + d = self.inter_data[index] + input, target = self._get_text_data(d, self.prompt) + + return dict(input_ids=input, labels=target) \ No newline at end of file diff --git a/data_process/amazon18_data_process.py b/data_process/amazon18_data_process.py new file mode 100644 index 0000000000000000000000000000000000000000..4dc45b5711d5632a4f115acf4ba656cfaab513dc --- /dev/null +++ b/data_process/amazon18_data_process.py @@ -0,0 +1,299 @@ +import argparse +import collections +import gzip +import html +import json +import os +import random +import re +import torch +from tqdm import tqdm +import numpy as np +from utils import check_path, clean_text, amazon18_dataset2fullname, write_json_file, write_remap_index + +def load_ratings(file): + users, items, inters = set(), set(), set() + with open(file, 'r') as fp: + for line in tqdm(fp, desc='Load ratings'): + try: + item, user, rating, time = line.strip().split(',') + users.add(user) + items.add(item) + inters.add((user, item, float(rating), int(time))) + except ValueError: + print(line) + return users, items, inters + + +def load_meta_items(file): + items = {} + with gzip.open(file, "r") as fp: + for line in tqdm(fp, desc="Load metas"): + data = json.loads(line) + item = data["asin"] + title = clean_text(data["title"]) + + descriptions = data["description"] + descriptions = clean_text(descriptions) + + brand = data["brand"].replace("by\n", "").strip() + + categories = data["category"] + new_categories = [] + for category in categories: + if "" in category: + break + new_categories.append(category.strip()) + categories = ",".join(new_categories).strip() + + items[item] = {"title": title, "description": descriptions, "brand": brand, "categories": categories} + # print(items[item]) + return items + + +def load_review_data(args, user2id, item2id): + + dataset_full_name = amazon18_dataset2fullname[args.dataset] + review_file_path = os.path.join(args.input_path, 'Review', dataset_full_name + '.json.gz') + + reviews = {} + + with gzip.open(review_file_path, "r") as fp: + + for line in tqdm(fp,desc='Load reviews'): + inter = json.loads(line) + try: + user = inter['reviewerID'] + item = inter['asin'] + if user in user2id and item in item2id: + uid = user2id[user] + iid = item2id[item] + else: + continue + if 'reviewText' in inter: + review = clean_text(inter['reviewText']) + else: + review = '' + if 'summary' in inter: + summary = clean_text(inter['summary']) + else: + summary = '' + reviews[str((uid,iid))]={"review":review, "summary":summary} + + except ValueError: + print(line) + + return reviews + + +def get_user2count(inters): + user2count = collections.defaultdict(int) + for unit in inters: + user2count[unit[0]] += 1 + return user2count + + +def get_item2count(inters): + item2count = collections.defaultdict(int) + for unit in inters: + item2count[unit[1]] += 1 + return item2count + + +def generate_candidates(unit2count, threshold): + cans = set() + for unit, count in unit2count.items(): + if count >= threshold: + cans.add(unit) + return cans, len(unit2count) - len(cans) + + +def filter_inters(inters, can_items=None, + user_k_core_threshold=0, item_k_core_threshold=0): + new_inters = [] + + # filter by meta items + if can_items: + print('\nFiltering by meta items: ') + for unit in inters: + if unit[1] in can_items.keys(): + new_inters.append(unit) + inters, new_inters = new_inters, [] + print(' The number of inters: ', len(inters)) + + # filter by k-core + if user_k_core_threshold or item_k_core_threshold: + print('\nFiltering by k-core:') + idx = 0 + user2count = get_user2count(inters) + item2count = get_item2count(inters) + + while True: + new_user2count = collections.defaultdict(int) + new_item2count = collections.defaultdict(int) + users, n_filtered_users = generate_candidates( # users is set + user2count, user_k_core_threshold) + items, n_filtered_items = generate_candidates( + item2count, item_k_core_threshold) + if n_filtered_users == 0 and n_filtered_items == 0: + break + for unit in inters: + if unit[0] in users and unit[1] in items: + new_inters.append(unit) + new_user2count[unit[0]] += 1 + new_item2count[unit[1]] += 1 + idx += 1 + inters, new_inters = new_inters, [] + user2count, item2count = new_user2count, new_item2count + print(' Epoch %d The number of inters: %d, users: %d, items: %d' + % (idx, len(inters), len(user2count), len(item2count))) + return inters + + +def make_inters_in_order(inters): + user2inters, new_inters = collections.defaultdict(list), list() + for inter in inters: + user, item, rating, timestamp = inter + user2inters[user].append((user, item, rating, timestamp)) + for user in user2inters: + user_inters = user2inters[user] + user_inters.sort(key=lambda d: d[3]) + interacted_item = set() + for inter in user_inters: + if inter[1] in interacted_item: # 过滤重复交互 + continue + interacted_item.add(inter[1]) + new_inters.append(inter) + return new_inters + + +def preprocess_rating(args): + dataset_full_name = amazon18_dataset2fullname[args.dataset] + + print('Process rating data: ') + print(' Dataset: ', args.dataset) + + # load ratings + rating_file_path = os.path.join(args.input_path, 'Ratings', dataset_full_name + '.csv') + rating_users, rating_items, rating_inters = load_ratings(rating_file_path) + + # load item IDs with meta data + meta_file_path = os.path.join(args.input_path, 'Metadata', f'meta_{dataset_full_name}.json.gz') + meta_items = load_meta_items(meta_file_path) + + # 1. Filter items w/o meta data; + # 2. K-core filtering; + print('The number of raw inters: ', len(rating_inters)) + + rating_inters = make_inters_in_order(rating_inters) + + rating_inters = filter_inters(rating_inters, can_items=meta_items, + user_k_core_threshold=args.user_k, + item_k_core_threshold=args.item_k) + + # sort interactions chronologically for each user + rating_inters = make_inters_in_order(rating_inters) + print('\n') + + # return: list of (user_ID, item_ID, rating, timestamp) + return rating_inters, meta_items + +def convert_inters2dict(inters): + user2items = collections.defaultdict(list) + user2index, item2index = dict(), dict() + for inter in inters: + user, item, rating, timestamp = inter + if user not in user2index: + user2index[user] = len(user2index) + if item not in item2index: + item2index[item] = len(item2index) + user2items[user2index[user]].append(item2index[item]) + return user2items, user2index, item2index + +def generate_data(args, rating_inters): + print('Split dataset: ') + print(' Dataset: ', args.dataset) + + # generate train valid temp + user2items, user2index, item2index = convert_inters2dict(rating_inters) + train_inters, valid_inters, test_inters = dict(), dict(), dict() + for u_index in range(len(user2index)): + inters = user2items[u_index] + # leave one out + train_inters[u_index] = [str(i_index) for i_index in inters[:-2]] + valid_inters[u_index] = [str(inters[-2])] + test_inters[u_index] = [str(inters[-1])] + assert len(user2items[u_index]) == len(train_inters[u_index]) + \ + len(valid_inters[u_index]) + len(test_inters[u_index]) + return user2items, train_inters, valid_inters, test_inters, user2index, item2index + +def convert_to_atomic_files(args, train_data, valid_data, test_data): + print('Convert dataset: ') + print(' Dataset: ', args.dataset) + uid_list = list(train_data.keys()) + uid_list.sort(key=lambda t: int(t)) + + with open(os.path.join(args.output_path, args.dataset, f'{args.dataset}.train.inter'), 'w') as file: + file.write('user_id:token\titem_id_list:token_seq\titem_id:token\n') + for uid in uid_list: + item_seq = train_data[uid] + seq_len = len(item_seq) + for target_idx in range(1, seq_len): + target_item = item_seq[-target_idx] + seq = item_seq[:-target_idx][-50:] + file.write(f'{uid}\t{" ".join(seq)}\t{target_item}\n') + + with open(os.path.join(args.output_path, args.dataset, f'{args.dataset}.valid.inter'), 'w') as file: + file.write('user_id:token\titem_id_list:token_seq\titem_id:token\n') + for uid in uid_list: + item_seq = train_data[uid][-50:] + target_item = valid_data[uid][0] + file.write(f'{uid}\t{" ".join(item_seq)}\t{target_item}\n') + + with open(os.path.join(args.output_path, args.dataset, f'{args.dataset}.test.inter'), 'w') as file: + file.write('user_id:token\titem_id_list:token_seq\titem_id:token\n') + for uid in uid_list: + item_seq = (train_data[uid] + valid_data[uid])[-50:] + target_item = test_data[uid][0] + file.write(f'{uid}\t{" ".join(item_seq)}\t{target_item}\n') + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--dataset', type=str, default='Arts', help='Instruments / Arts / Games') + parser.add_argument('--user_k', type=int, default=5, help='user k-core filtering') + parser.add_argument('--item_k', type=int, default=5, help='item k-core filtering') + parser.add_argument('--input_path', type=str, default='') + parser.add_argument('--output_path', type=str, default='') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + + # load interactions from raw rating file + rating_inters, meta_items = preprocess_rating(args) + + + # split train/valid/temp + all_inters,train_inters, valid_inters, test_inters, user2index, item2index = generate_data(args, rating_inters) + + check_path(os.path.join(args.output_path, args.dataset)) + + write_json_file(all_inters, os.path.join(args.output_path, args.dataset, f'{args.dataset}.inter.json')) + convert_to_atomic_files(args, train_inters, valid_inters, test_inters) + + item2feature = collections.defaultdict(dict) + for item, item_id in item2index.items(): + item2feature[item_id] = meta_items[item] + + # reviews = load_review_data(args, user2index, item2index) + + print("user:",len(user2index)) + print("item:",len(item2index)) + + write_json_file(item2feature, os.path.join(args.output_path, args.dataset, f'{args.dataset}.item.json')) + # write_json_file(reviews, os.path.join(args.output_path, args.dataset, f'{args.dataset}.review.json')) + + + write_remap_index(user2index, os.path.join(args.output_path, args.dataset, f'{args.dataset}.user2id')) + write_remap_index(item2index, os.path.join(args.output_path, args.dataset, f'{args.dataset}.item2id')) \ No newline at end of file diff --git a/data_process/amazon18_recbole_data_process.py b/data_process/amazon18_recbole_data_process.py new file mode 100644 index 0000000000000000000000000000000000000000..45aa42ae21ed41ff60e91db5b1f8019efcc90ef1 --- /dev/null +++ b/data_process/amazon18_recbole_data_process.py @@ -0,0 +1,226 @@ +import argparse +import collections +import gzip +import html +import json +import os +import random +import re +import torch +from tqdm import tqdm +import numpy as np +from utils import check_path, clean_text, amazon18_dataset2fullname,write_json_file,write_remap_index + +def load_ratings(file): + users, items, inters = set(), set(), set() + with open(file, 'r') as fp: + for line in tqdm(fp, desc='Load ratings'): + try: + item, user, rating, time = line.strip().split(',') + users.add(user) + items.add(item) + inters.add((user, item, float(rating), int(time))) + except ValueError: + print(line) + return users, items, inters + + +def load_meta_items(file): + items = {} + # re_tag = re.compile(']*>') + with gzip.open(file, "r") as fp: + for line in tqdm(fp, desc="Load metas"): + data = json.loads(line) + item = data["asin"] + title = clean_text(data["title"]) + + descriptions = data["description"] + descriptions = clean_text(descriptions) + # new_descriptions = [] + # for description in descriptions: + # description = re.sub(re_tag, '', description) + # new_descriptions.append(description.strip()) + # descriptions = " ".join(new_descriptions).strip() + + brand = data["brand"].replace("by\n", "").strip() + + categories = data["category"] + new_categories = [] + for category in categories: + if "" in category: + break + new_categories.append(category.strip()) + categories = ",".join(new_categories[1:]).strip() + + items[item] = {"title": title, "description": descriptions, "brand": brand, "categories": categories} + # print(items[item]) + return items + + +def get_user2count(inters): + user2count = collections.defaultdict(int) + for unit in inters: + user2count[unit[0]] += 1 + return user2count + + +def get_item2count(inters): + item2count = collections.defaultdict(int) + for unit in inters: + item2count[unit[1]] += 1 + return item2count + + +def generate_candidates(unit2count, threshold): + cans = set() + for unit, count in unit2count.items(): + if count >= threshold: + cans.add(unit) + return cans, len(unit2count) - len(cans) + + +def filter_inters(inters, can_items=None, + user_k_core_threshold=0, item_k_core_threshold=0): + new_inters = [] + + # filter by meta items + if can_items: + print('\nFiltering by meta items: ') + for unit in inters: + if unit[1] in can_items.keys(): + new_inters.append(unit) + inters, new_inters = new_inters, [] + print(' The number of inters: ', len(inters)) + + # filter by k-core + if user_k_core_threshold or item_k_core_threshold: + print('\nFiltering by k-core:') + idx = 0 + user2count = get_user2count(inters) + item2count = get_item2count(inters) + + while True: + new_user2count = collections.defaultdict(int) + new_item2count = collections.defaultdict(int) + users, n_filtered_users = generate_candidates( # users is set + user2count, user_k_core_threshold) + items, n_filtered_items = generate_candidates( + item2count, item_k_core_threshold) + if n_filtered_users == 0 and n_filtered_items == 0: + break + for unit in inters: + if unit[0] in users and unit[1] in items: + new_inters.append(unit) + new_user2count[unit[0]] += 1 + new_item2count[unit[1]] += 1 + idx += 1 + inters, new_inters = new_inters, [] + user2count, item2count = new_user2count, new_item2count + print(' Epoch %d The number of inters: %d, users: %d, items: %d' + % (idx, len(inters), len(user2count), len(item2count))) + return inters + + +def make_inters_in_order(inters): + user2inters, new_inters = collections.defaultdict(list), list() + for inter in inters: + user, item, rating, timestamp = inter + user2inters[user].append((user, item, rating, timestamp)) + for user in user2inters: + user_inters = user2inters[user] + user_inters.sort(key=lambda d: d[3]) + interacted_item = set() + for inter in user_inters: + if inter[1] in interacted_item: # 过滤重复交互 + continue + interacted_item.add(inter[1]) + new_inters.append(inter) + return new_inters + + +def preprocess_rating(args): + dataset_full_name = amazon18_dataset2fullname[args.dataset] + + print('Process rating data: ') + print(' Dataset: ', args.dataset) + + # load ratings + rating_file_path = os.path.join(args.input_path, 'Ratings', dataset_full_name + '.csv') + rating_users, rating_items, rating_inters = load_ratings(rating_file_path) + + # load item IDs with meta data + meta_file_path = os.path.join(args.input_path, 'Metadata', f'meta_{dataset_full_name}.json.gz') + meta_items = load_meta_items(meta_file_path) + + # 1. Filter items w/o meta data; + # 2. K-core filtering; + print('The number of raw inters: ', len(rating_inters)) + + rating_inters = make_inters_in_order(rating_inters) + + rating_inters = filter_inters(rating_inters, can_items=meta_items, + user_k_core_threshold=args.user_k, + item_k_core_threshold=args.item_k) + + # sort interactions chronologically for each user + rating_inters = make_inters_in_order(rating_inters) + print('\n') + + # return: list of (user_ID, item_ID, rating, timestamp) + return rating_inters, meta_items + +def save_inter(args, inters): + print('Convert dataset: ') + print(' Dataset: ', args.dataset) + + with open(os.path.join(args.output_path, args.dataset, f'{args.dataset}.inter'), 'w') as file: + file.write('user_id:token\titem_id:token\trating:float\ttimestamp:float\n') + for inter in inters: + user, item, rating, timestamp = inter + file.write(f'{user}\t{item}\t{rating}\t{timestamp}\n') + + +def save_feat(args, feat, all_items): + iid_list = list(feat.keys()) + num_item = 0 + with open(os.path.join(args.output_path, args.dataset, f'{args.dataset}.item'), 'w') as file: + # "title": title, "description": descriptions, "brand": brand, "categories": categories + file.write('item_id:token\ttitle:token_seq\tbrand:token\tcategories:token_seq\n') + for iid in iid_list: + if iid in all_items: + num_item += 1 + title, brand, categories = feat[iid]["title"], feat[iid]["brand"], feat[iid]["categories"] + file.write(f'{iid}\t{title}\t{brand}\t{categories}\n') + print("num_item: ", num_item) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--dataset', type=str, default='Arts', help='Instruments / Arts / Games') + parser.add_argument('--user_k', type=int, default=5, help='user k-core filtering') + parser.add_argument('--item_k', type=int, default=5, help='item k-core filtering') + parser.add_argument('--input_path', type=str, default='') + parser.add_argument('--output_path', type=str, default='') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + + # load interactions from raw rating file + rating_inters, meta_items = preprocess_rating(args) + + check_path(os.path.join(args.output_path, args.dataset)) + + + all_items = set() + for inter in rating_inters: + user, item, rating, timestamp = inter + all_items.add(item) + + print("total item: ", len(list(all_items))) + + save_inter(args,rating_inters) + save_feat(args,meta_items, all_items) + + diff --git a/data_process/amazon_text_emb.py b/data_process/amazon_text_emb.py new file mode 100644 index 0000000000000000000000000000000000000000..7818592a5e96d380cc232c0c3b2d4b95b4e93a71 --- /dev/null +++ b/data_process/amazon_text_emb.py @@ -0,0 +1,139 @@ +import argparse +import collections +import gzip +import html +import json +import os +import random +import re +import torch +from tqdm import tqdm +import numpy as np +from utils import * +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig, AutoTokenizer, AutoModel + + +def load_data(args): + + item2feature_path = os.path.join(args.root, f'{args.dataset}.item.json') + item2feature = load_json(item2feature_path) + + return item2feature + +def generate_text(item2feature, features): + item_text_list = [] + + for item in item2feature: + data = item2feature[item] + text = [] + for meta_key in features: + if meta_key in data: + meta_value = clean_text(data[meta_key]) + text.append(meta_value.strip()) + + item_text_list.append([int(item), text]) + + return item_text_list + +def preprocess_text(args): + print('Process text data: ') + print(' Dataset: ', args.dataset) + + item2feature = load_data(args) + # load item text and clean + item_text_list = generate_text(item2feature, ['title', 'description']) + # item_text_list = generate_text(item2feature, ['title']) + # return: list of (item_ID, cleaned_item_text) + return item_text_list + +def generate_item_embedding(args, item_text_list, tokenizer, model, word_drop_ratio=-1): + print(f'Generate Text Embedding: ') + print(' Dataset: ', args.dataset) + + items, texts = zip(*item_text_list) + order_texts = [[0]] * len(items) + for item, text in zip(items, texts): + order_texts[item] = text + for text in order_texts: + assert text != [0] + + embeddings = [] + start, batch_size = 0, 1 + with torch.no_grad(): + while start < len(order_texts): + if (start+1)%100==0: + print("==>",start+1) + field_texts = order_texts[start: start + batch_size] + # print(field_texts) + field_texts = zip(*field_texts) + + field_embeddings = [] + for sentences in field_texts: + sentences = list(sentences) + # print(sentences) + if word_drop_ratio > 0: + print(f'Word drop with p={word_drop_ratio}') + new_sentences = [] + for sent in sentences: + new_sent = [] + sent = sent.split(' ') + for wd in sent: + rd = random.random() + if rd > word_drop_ratio: + new_sent.append(wd) + new_sent = ' '.join(new_sent) + new_sentences.append(new_sent) + sentences = new_sentences + encoded_sentences = tokenizer(sentences, max_length=args.max_sent_len, + truncation=True, return_tensors='pt',padding="longest").to(args.device) + outputs = model(input_ids=encoded_sentences.input_ids, + attention_mask=encoded_sentences.attention_mask) + + masked_output = outputs.last_hidden_state * encoded_sentences['attention_mask'].unsqueeze(-1) + mean_output = masked_output.sum(dim=1) / encoded_sentences['attention_mask'].sum(dim=-1, keepdim=True) + mean_output = mean_output.detach().cpu() + field_embeddings.append(mean_output) + + field_mean_embedding = torch.stack(field_embeddings, dim=0).mean(dim=0) + embeddings.append(field_mean_embedding) + start += batch_size + + embeddings = torch.cat(embeddings, dim=0).numpy() + print('Embeddings shape: ', embeddings.shape) + + file = os.path.join(args.root, args.dataset + '.emb-' + args.plm_name + "-td" + ".npy") + np.save(file, embeddings) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--dataset', type=str, default='Arts', help='Instruments / Arts / Games') + parser.add_argument('--root', type=str, default="") + parser.add_argument('--gpu_id', type=int, default=2, help='ID of running GPU') + parser.add_argument('--plm_name', type=str, default='llama') + parser.add_argument('--plm_checkpoint', type=str, + default='') + parser.add_argument('--max_sent_len', type=int, default=2048) + parser.add_argument('--word_drop_ratio', type=float, default=-1, help='word drop ratio, do not drop by default') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + + args.root = os.path.join(args.root, args.dataset) + + device = set_device(args.gpu_id) + args.device = device + + item_text_list = preprocess_text(args) + + plm_tokenizer, plm_model = load_plm(args.plm_checkpoint) + if plm_tokenizer.pad_token_id is None: + plm_tokenizer.pad_token_id = 0 + plm_model = plm_model.to(device) + + generate_item_embedding(args, item_text_list,plm_tokenizer, + plm_model, word_drop_ratio=args.word_drop_ratio) + + diff --git a/data_process/get_llm_output.py b/data_process/get_llm_output.py new file mode 100644 index 0000000000000000000000000000000000000000..a7a7cd1ee8e9710319cfd16cdb09283ebd59b699 --- /dev/null +++ b/data_process/get_llm_output.py @@ -0,0 +1,374 @@ + + +import argparse +import os +import os.path as osp +import random +import time +from logging import getLogger +import openai +from utils import get_res_batch, load_json, intention_prompt, preference_prompt_1, preference_prompt_2, amazon18_dataset2fullname, write_json_file +import json + + + +def get_intention_train(args, inters, item2feature, reviews, api_info): + + intention_train_output_file = os.path.join(args.root,"intention_train.json") + + + # Suggest modifying the prompt based on different datasets + prompt = intention_prompt + dataset_full_name = amazon18_dataset2fullname[args.dataset] + dataset_full_name = dataset_full_name.replace("_", " ").lower() + print(dataset_full_name) + + prompt_list = [] + + inter_data = [] + + for (user,item_list) in inters.items(): + user = int(user) + item = int(item_list[-3]) + history = item_list[:-3] + + inter_data.append((user,item,history)) + + review = reviews[str((user, item))]["review"] + item_title = item2feature[str(item)]["title"] + input_prompt = prompt.format(item_title=item_title,dataset_full_name=dataset_full_name,review=review) + prompt_list.append(input_prompt) + + st = 0 + with open(intention_train_output_file, mode='a') as f: + + while st < len(prompt_list): + # while st < 3: + print(st) + # if st < 25631: + # st += args.batchsize + # continue + + + res = get_res_batch(args.model_name, prompt_list[st:st+args.batchsize], args.max_tokens, api_info) + + for i, answer in enumerate(res): + user, item, history = inter_data[st+i] + # print(answer) + # print("=============") + + if answer == '': + print("answer null error") + answer = "I enjoy high-quality item." + + if answer.strip().count('\n') != 1: + if 'haracteristics:' in answer: + answer = answer.strip().split("The item's characteristics:") + else: + answer = answer.strip().split("The item's characteristic:") + else: + answer = answer.strip().split('\n') + + if '' in answer: + answer.remove('') + + if len(answer) == 1: + print(answer) + user_preference = item_character = answer[0] + elif len(answer) >= 3: + print(answer) + answer = answer[-1] + user_preference = item_character = answer + else: + user_preference, item_character = answer + + if ':' in user_preference: + idx = user_preference.index(':') + user_preference = user_preference[idx+1:] + user_preference = user_preference.strip().replace('}','') + user_preference = user_preference.replace('\n','') + + if ':' in item_character: + idx = item_character.index(':') + item_character = item_character[idx+1:] + item_character = item_character.strip().replace('}','') + item_character = item_character.replace('\n','') + + + dict = {"user":user, "item":item, "inters": history, + "user_related_intention":user_preference, "item_related_intention": item_character} + + json.dump(dict, f) + f.write("\n") + + st += args.batchsize + + return intention_train_output_file + + +def get_intention_test(args, inters, item2feature, reviews, api_info): + + intention_test_output_file = os.path.join(args.root,"intention_test.json") + + # Suggest modifying the prompt based on different datasets + prompt = intention_prompt + dataset_full_name = amazon18_dataset2fullname[args.dataset] + dataset_full_name = dataset_full_name.replace("_", " ").lower() + print(dataset_full_name) + + prompt_list = [] + + inter_data = [] + + for (user,item_list) in inters.items(): + user = int(user) + item = int(item_list[-1]) + history = item_list[:-1] + + inter_data.append((user,item,history)) + + review = reviews[str((user, item))]["review"] + item_title = item2feature[str(item)]["title"] + input_prompt = prompt.format(item_title=item_title,dataset_full_name=dataset_full_name,review=review) + prompt_list.append(input_prompt) + + st = 0 + with open(intention_test_output_file, mode='a') as f: + + while st < len(prompt_list): + # while st < 3: + print(st) + # if st < 4623: + # st += args.batchsize + # continue + + res = get_res_batch(args.model_name, prompt_list[st:st+args.batchsize], args.max_tokens, api_info) + + for i, answer in enumerate(res): + user, item, history = inter_data[st+i] + + if answer == '': + print("answer null error") + answer = "I enjoy high-quality item." + + if answer.strip().count('\n') != 1: + if 'haracteristics:' in answer: + answer = answer.strip().split("The item's characteristics:") + else: + answer = answer.strip().split("The item's characteristic:") + else: + answer = answer.strip().split('\n') + + if '' in answer: + answer.remove('') + + if len(answer) == 1: + print(answer) + user_preference = item_character = answer[0] + elif len(answer) >= 3: + print(answer) + answer = answer[-1] + user_preference = item_character = answer + else: + user_preference, item_character = answer + + if ':' in user_preference: + idx = user_preference.index(':') + user_preference = user_preference[idx+1:] + user_preference = user_preference.strip().replace('}','') + user_preference = user_preference.replace('\n','') + + if ':' in item_character: + idx = item_character.index(':') + item_character = item_character[idx+1:] + item_character = item_character.strip().replace('}','') + item_character = item_character.replace('\n','') + + + dict = {"user":user, "item":item, "inters": history, + "user_related_intention":user_preference, "item_related_intention": item_character} + + json.dump(dict, f) + f.write("\n") + + st += args.batchsize + + return intention_test_output_file + + + + +def get_user_preference(args, inters, item2feature, reviews, api_info): + + preference_output_file = os.path.join(args.root,"user_preference.json") + + + # Suggest modifying the prompt based on different datasets + prompt_1 = preference_prompt_1 + prompt_2 = preference_prompt_2 + + + dataset_full_name = amazon18_dataset2fullname[args.dataset] + dataset_full_name = dataset_full_name.replace("_", " ").lower() + print(dataset_full_name) + + prompt_list_1 = [] + prompt_list_2 = [] + + users = [] + + for (user,item_list) in inters.items(): + users.append(user) + history = item_list[:-3] + item_titles = [] + for j, item in enumerate(history): + item_titles.append(str(j+1) + '.' + item2feature[str(item)]["title"]) + if len(item_titles) > args.max_his_len: + item_titles = item_titles[-args.max_his_len:] + item_titles = ", ".join(item_titles) + + input_prompt_1 = prompt_1.format(dataset_full_name=dataset_full_name, item_titles=item_titles) + input_prompt_2 = prompt_2.format(dataset_full_name=dataset_full_name, item_titles=item_titles) + + prompt_list_1.append(input_prompt_1) + prompt_list_2.append(input_prompt_2) + + + st = 0 + with open(preference_output_file, mode='a') as f: + + while st < len(prompt_list_1): + # while st < 3: + print(st) + # if st < 22895: + # st += args.batchsize + # continue + + res_1 = get_res_batch(args.model_name, prompt_list_1[st:st + args.batchsize], args.max_tokens, api_info) + res_2 = get_res_batch(args.model_name, prompt_list_2[st:st + args.batchsize], args.max_tokens, api_info) + for i, answers in enumerate(zip(res_1, res_2)): + + user = users[st + i] + + answer_1, answer_2 = answers + # print(answers) + # print("=============") + + if answer_1 == '': + print("answer null error") + answer_1 = "I enjoy high-quality item." + + if answer_2 == '': + print("answer null error") + answer_2 = "I enjoy high-quality item." + + if answer_2.strip().count('\n') != 1: + if 'references:' in answer_2: + answer_2 = answer_2.strip().split("Short-term preferences:") + else: + answer_2 = answer_2.strip().split("Short-term preference:") + else: + answer_2 = answer_2.strip().split('\n') + + if '' in answer_2: + answer_2.remove('') + + if len(answer_2) == 1: + print(answer_2) + long_preference = short_preference = answer_2[0] + elif len(answer_2) >= 3: + print(answer_2) + answer_2 = answer_2[-1] + long_preference = short_preference = answer_2 + else: + long_preference, short_preference = answer_2 + + if ':' in long_preference: + idx = long_preference.index(':') + long_preference = long_preference[idx+1:] + long_preference = long_preference.strip().replace('}','') + long_preference = long_preference.replace('\n','') + + if ':' in short_preference: + idx = short_preference.index(':') + short_preference = short_preference[idx+1:] + short_preference = short_preference.strip().replace('}','') + short_preference = short_preference.replace('\n','') + + dict = {"user":user,"user_preference":[answer_1, long_preference, short_preference]} + # print(dict) + json.dump(dict, f) + f.write("\n") + + st += args.batchsize + + return preference_output_file + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--dataset', type=str, default='Instruments', help='Instruments / Arts / Games') + parser.add_argument('--root', type=str, default='') + parser.add_argument('--api_info', type=str, default='./api_info.json') + parser.add_argument('--model_name', type=str, default='text-davinci-003') + parser.add_argument('--max_tokens', type=int, default=512) + parser.add_argument('--batchsize', type=int, default=16) + parser.add_argument('--max_his_len', type=int, default=20) + return parser.parse_args() + +if __name__ == "__main__": + args = parse_args() + + args.root = os.path.join(args.root, args.dataset) + + api_info = load_json(args.api_info) + openai.api_key = api_info["api_key_list"].pop() + + + inter_path = os.path.join(args.root, f'{args.dataset}.inter.json') + inters = load_json(inter_path) + + + item2feature_path = os.path.join(args.root, f'{args.dataset}.item.json') + item2feature = load_json(item2feature_path) + + reviews_path = os.path.join(args.root, f'{args.dataset}.review.json') + reviews = load_json(reviews_path) + + intention_train_output_file = get_intention_train(args, inters, item2feature, reviews, api_info) + intention_test_output_file = get_intention_test(args, inters, item2feature, reviews ,api_info) + preference_output_file = get_user_preference(args, inters, item2feature, reviews, api_info) + + intention_train = {} + intention_test = {} + user_preference = {} + + with open(intention_train_output_file, "r") as f: + for line in f: + # print(line) + content = json.loads(line) + if content["user"] not in intention_train: + intention_train[content["user"]] = {"item":content["item"], + "inters":content["inters"], + "querys":[ content["user_related_intention"], content["item_related_intention"] ]} + + + with open(intention_test_output_file, "r") as f: + for line in f: + content = json.loads(line) + if content["user"] not in intention_train: + intention_test[content["user"]] = {"item":content["item"], + "inters":content["inters"], + "querys":[ content["user_related_intention"], content["item_related_intention"] ]} + + + with open(preference_output_file, "r") as f: + for line in f: + content = json.loads(line) + user_preference[content["user"]] = content["user_preference"] + + user_dict = { + "user_explicit_preference": user_preference, + "user_vague_intention": {"train": intention_train, "test": intention_test}, + } + + write_json_file(user_dict, os.path.join(args.root, f'{args.dataset}.user.json')) diff --git a/data_process/utils.py b/data_process/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c32b5f588d8f124be50b33c26ad499643151df9d --- /dev/null +++ b/data_process/utils.py @@ -0,0 +1,238 @@ +import html +import json +import os +import pickle +import re +import time + +import torch +# import gensim +from transformers import AutoModel, AutoTokenizer +import collections +import openai + + + +def get_res_batch(model_name, prompt_list, max_tokens, api_info): + + while True: + try: + res = openai.Completion.create( + model=model_name, + prompt=prompt_list, + temperature=0.4, + max_tokens=max_tokens, + top_p=1, + frequency_penalty=0, + presence_penalty=0 + ) + output_list = [] + for choice in res['choices']: + output = choice['text'].strip() + output_list.append(output) + + return output_list + + except openai.error.AuthenticationError as e: + print(e) + openai.api_key = api_info["api_key_list"].pop() + time.sleep(10) + except openai.error.RateLimitError as e: + print(e) + if str(e) == "You exceeded your current quota, please check your plan and billing details.": + openai.api_key = api_info["api_key_list"].pop() + time.sleep(10) + else: + print('\nopenai.error.RateLimitError\nRetrying...') + time.sleep(10) + except openai.error.ServiceUnavailableError as e: + print(e) + print('\nopenai.error.ServiceUnavailableError\nRetrying...') + time.sleep(10) + except openai.error.Timeout: + print('\nopenai.error.Timeout\nRetrying...') + time.sleep(10) + except openai.error.APIError as e: + print(e) + print('\nopenai.error.APIError\nRetrying...') + time.sleep(10) + except openai.error.APIConnectionError as e: + print(e) + print('\nopenai.error.APIConnectionError\nRetrying...') + time.sleep(10) + except Exception as e: + print(e) + return None + + + + +def check_path(path): + if not os.path.exists(path): + os.makedirs(path) + + +def set_device(gpu_id): + if gpu_id == -1: + return torch.device('cpu') + else: + return torch.device( + 'cuda:' + str(gpu_id) if torch.cuda.is_available() else 'cpu') + +def load_plm(model_path='bert-base-uncased'): + + tokenizer = AutoTokenizer.from_pretrained(model_path,) + + print("Load Model:", model_path) + + model = AutoModel.from_pretrained(model_path,low_cpu_mem_usage=True,) + return tokenizer, model + +def load_json(file): + with open(file, 'r') as f: + data = json.load(f) + return data + +def clean_text(raw_text): + if isinstance(raw_text, list): + new_raw_text=[] + for raw in raw_text: + raw = html.unescape(raw) + raw = re.sub(r']*>', '', raw) + raw = re.sub(r'["\n\r]*', '', raw) + new_raw_text.append(raw.strip()) + cleaned_text = ' '.join(new_raw_text) + else: + if isinstance(raw_text, dict): + cleaned_text = str(raw_text)[1:-1].strip() + else: + cleaned_text = raw_text.strip() + cleaned_text = html.unescape(cleaned_text) + cleaned_text = re.sub(r']*>', '', cleaned_text) + cleaned_text = re.sub(r'["\n\r]*', '', cleaned_text) + index = -1 + while -index < len(cleaned_text) and cleaned_text[index] == '.': + index -= 1 + index += 1 + if index == 0: + cleaned_text = cleaned_text + '.' + else: + cleaned_text = cleaned_text[:index] + '.' + if len(cleaned_text) >= 2000: + cleaned_text = '' + return cleaned_text + +def load_pickle(filename): + with open(filename, "rb") as f: + return pickle.load(f) + + +def make_inters_in_order(inters): + user2inters, new_inters = collections.defaultdict(list), list() + for inter in inters: + user, item, rating, timestamp = inter + user2inters[user].append((user, item, rating, timestamp)) + for user in user2inters: + user_inters = user2inters[user] + user_inters.sort(key=lambda d: d[3]) + for inter in user_inters: + new_inters.append(inter) + return new_inters + +def write_json_file(dic, file): + print('Writing json file: ',file) + with open(file, 'w') as fp: + json.dump(dic, fp, indent=4) + +def write_remap_index(unit2index, file): + print('Writing remap file: ',file) + with open(file, 'w') as fp: + for unit in unit2index: + fp.write(unit + '\t' + str(unit2index[unit]) + '\n') + + +intention_prompt = "After purchasing a {dataset_full_name} item named \"{item_title}\", the user left a comment expressing his opinion and personal preferences. The user's comment is as follows: \n\"{review}\" " \ + "\nAs we all know, user comments often contain information about both their personal preferences and the characteristics of the item they interacted with. From this comment, you can infer both the user's personal preferences and the characteristics of the item. " \ + "Please describe your inferred user preferences and item characteristics in the first person and in the following format:\n\nMy preferences: []\nThe item's characteristics: []\n\n" \ + "Note that your inference of the personalized preferences should not include any information about the title of the item." + + +preference_prompt_1 = "Suppose the user has bought a variety of {dataset_full_name} items, they are: \n{item_titles}. \nAs we all know, these historically purchased items serve as a reflection of the user's personalized preferences. " \ + "Please analyze the user's personalized preferences based on the items he has bought and provide a brief third-person summary of the user's preferences, highlighting the key factors that influence his choice of items. Avoid listing specific items and do not list multiple examples. " \ + "Your analysis should be brief and in the third person." + +preference_prompt_2 = "Given a chronological list of {dataset_full_name} items that a user has purchased, we can analyze his long-term and short-term preferences. Long-term preferences are inherent characteristics of the user, which are reflected in all the items he has interacted with over time. Short-term preferences are the user's recent preferences, which are reflected in some of the items he has bought more recently. " \ + "To determine the user's long-term preferences, please analyze the contents of all the items he has bought. Look for common features that appear frequently across the user's shopping records. To determine the user's short-term preferences, focus on the items he has bought most recently. Identify any new or different features that have emerged in the user's shopping records. " \ + "Here is a chronological list of items that the user has bought: \n{item_titles}. \nPlease provide separate analyses for the user's long-term and short-term preferences. Your answer should be concise and general, without listing specific items. Your answer should be in the third person and in the following format:\n\nLong-term preferences: []\nShort-term preferences: []\n\n" + + +# remove 'Magazine', 'Gift', 'Music', 'Kindle' +amazon18_dataset_list = [ + 'Appliances', 'Beauty', + 'Fashion', 'Software', 'Luxury', 'Scientific', 'Pantry', + 'Instruments', 'Arts', 'Games', 'Office', 'Garden', + 'Food', 'Cell', 'CDs', 'Automotive', 'Toys', + 'Pet', 'Tools', 'Kindle', 'Sports', 'Movies', + 'Electronics', 'Home', 'Clothing', 'Books' +] + +amazon18_dataset2fullname = { + 'Beauty': 'All_Beauty', + 'Fashion': 'AMAZON_FASHION', + 'Appliances': 'Appliances', + 'Arts': 'Arts_Crafts_and_Sewing', + 'Automotive': 'Automotive', + 'Books': 'Books', + 'CDs': 'CDs_and_Vinyl', + 'Cell': 'Cell_Phones_and_Accessories', + 'Clothing': 'Clothing_Shoes_and_Jewelry', + 'Music': 'Digital_Music', + 'Electronics': 'Electronics', + 'Gift': 'Gift_Cards', + 'Food': 'Grocery_and_Gourmet_Food', + 'Home': 'Home_and_Kitchen', + 'Scientific': 'Industrial_and_Scientific', + 'Kindle': 'Kindle_Store', + 'Luxury': 'Luxury_Beauty', + 'Magazine': 'Magazine_Subscriptions', + 'Movies': 'Movies_and_TV', + 'Instruments': 'Musical_Instruments', + 'Office': 'Office_Products', + 'Garden': 'Patio_Lawn_and_Garden', + 'Pet': 'Pet_Supplies', + 'Pantry': 'Prime_Pantry', + 'Software': 'Software', + 'Sports': 'Sports_and_Outdoors', + 'Tools': 'Tools_and_Home_Improvement', + 'Toys': 'Toys_and_Games', + 'Games': 'Video_Games' +} + +amazon14_dataset_list = [ + 'Beauty','Toys','Sports' +] + +amazon14_dataset2fullname = { + 'Beauty': 'Beauty', + 'Sports': 'Sports_and_Outdoors', + 'Toys': 'Toys_and_Games', +} + +# c1. c2. c3. c4. +amazon_text_feature1 = ['title', 'category', 'brand'] + +# re-order +amazon_text_feature1_ro1 = ['brand', 'main_cat', 'category', 'title'] + +# remove +amazon_text_feature1_re1 = ['title'] + +amazon_text_feature2 = ['title'] + +amazon_text_feature3 = ['description'] + +amazon_text_feature4 = ['description', 'main_cat', 'category', 'brand'] + +amazon_text_feature5 = ['title', 'description'] + + diff --git a/evaluate-finetuned.py b/evaluate-finetuned.py new file mode 100644 index 0000000000000000000000000000000000000000..46e890a75a156eea5909ee3b42c21a6b1fd21279 --- /dev/null +++ b/evaluate-finetuned.py @@ -0,0 +1,216 @@ +import argparse +import json +import os +import sys + +import torch +import transformers +import torch.distributed as dist +from torch.utils.data.distributed import DistributedSampler +from torch.nn.parallel import DistributedDataParallel +from peft import PeftModel +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig + +from utils import * +from collator import TestCollator +from prompt import all_prompt +from evaluate import get_topk_results, get_metrics_results + +parser = argparse.ArgumentParser(description = 'rqllama-evaluate') +parser = parse_evaluate_args(parser) +args = parser.parse_args() + +set_seed(args.seed) +world_size = int(os.environ.get("WORLD_SIZE", 1)) +local_rank = int(os.environ.get("LOCAL_RANK") or 0) +torch.cuda.set_device(local_rank) +if local_rank == 0: + print(vars(args)) + +dist.init_process_group(backend = "nccl", world_size = world_size, rank = local_rank) + +device_map = {"": local_rank} +device = torch.device("cuda",local_rank) + +tokenizer = LlamaTokenizer.from_pretrained(args.ckpt_path) +base_model = LlamaForCausalLM.from_pretrained(args.base_model, torch_dtype=torch.float16, low_cpu_mem_usage = True, device_map = device_map) +base_model.resize_token_embeddings(len(tokenizer)) +model = PeftModel.from_pretrained(base_model, args.ckpt_path, torch_dtype = torch.float16, device_map = device_map) + +model = DistributedDataParallel(model, device_ids = [local_rank]) + +if args.test_prompt_ids == "all": + if args.test_task.lower() == "seqrec": + prompt_ids = range(len(all_prompt["seqrec"])) + elif args.test_task.lower() == "itemsearch": + prompt_ids = range(len(all_prompt["itemsearch"])) + elif args.test_task.lower() == "fusionseqrec": + prompt_ids = range(len(all_prompt["fusionseqrec"])) +else: + prompt_ids = [int(_) for _ in args.test_prompt_ids.split(",")] + +test_data = load_test_dataset(args) +if local_rank == 0: + print("evaluate data num:", len(test_data)) +ddp_sampler = DistributedSampler(test_data, num_replicas = world_size, rank = local_rank, drop_last = True) +collator = TestCollator(args, tokenizer) +all_items = test_data.get_all_items() +prefix_allowed_tokens = test_data.get_prefix_allowed_tokens_fn(tokenizer) +test_loader = DataLoader( + test_data, + batch_size = args.test_batch_size, + collate_fn = collator, + sampler = ddp_sampler, + num_workers = 4, + pin_memory = True +) + +model.eval() + +metrics = args.metrics.split(",") +all_prompt_results = [] + +print('prompts:', len(prompt_ids)) + +with torch.no_grad(): + for prompt_id in prompt_ids[:3]: + if local_rank == 0: + print("Start prompt: ",prompt_id) + test_loader.dataset.set_prompt(prompt_id) + metrics_results = {} + total = 0 + + for step, batch in enumerate(tqdm(test_loader)): + inputs = batch[0].to(device) + targets = batch[1] + bs = len(targets) + num_beams = args.num_beams + + while True: + try: + output = model.module.generate( + input_ids = inputs["input_ids"], + attention_mask = inputs["attention_mask"], + max_new_tokens = 10, + prefix_allowed_tokens_fn = prefix_allowed_tokens, + num_beams = num_beams, + num_return_sequences = num_beams, + output_scores = True, + return_dict_in_generate = True, + early_stopping = True, + ) + break + except torch.cuda.OutOfMemoryError as e: + print("Out of memory!") + num_beams = num_beams -1 + print("Beam:", num_beams) + except Exception: + raise RuntimeError + output_ids = output["sequences"] + scores = output["sequences_scores"] + + # output_ids.shape: torch.Size([20, 101]) + # scores.shape: torch.Size([20]) + + output = tokenizer.batch_decode(output_ids, skip_special_tokens = True) + # output.length: 20 + ''' + Below is an instruction that describes a task. + Write a response that appropriately completes the request.\n\n + ### Instruction:\nThe user has interacted with items , , + , in chronological order. + Can you predict the next possible item that the user may expect?\n\n + ### Response: + ''' + + topk_res = get_topk_results( + output, + scores, + targets, + num_beams, + all_items = all_items if args.filter_items else None + ) + + bs_gather_list = [None for _ in range(world_size)] + dist.all_gather_object(obj=bs, object_list=bs_gather_list) + total += sum(bs_gather_list) + res_gather_list = [None for _ in range(world_size)] + dist.all_gather_object(obj=topk_res, object_list=res_gather_list) + + if local_rank == 0: + all_device_topk_res = [] + for ga_res in res_gather_list: + all_device_topk_res += ga_res + batch_metrics_res = get_metrics_results(all_device_topk_res, metrics) + for m, res in batch_metrics_res.items(): + if m not in metrics_results: + metrics_results[m] = res + else: + metrics_results[m] += res + + if (step + 1) % 50 == 0: + temp = {} + for m in metrics_results: + temp[m] = metrics_results[m] / total + print(temp) + dist.barrier() + + if local_rank == 0: + for m in metrics_results: + metrics_results[m] = metrics_results[m] / total + all_prompt_results.append(metrics_results) + print("======================================================") + print("Prompt {} results: ".format(prompt_id), metrics_results) + print("======================================================") + print("") + dist.barrier() +dist.barrier() + +if local_rank == 0: + mean_results = {} + min_results = {} + max_results = {} + + for m in metrics: + all_res = [_[m] for _ in all_prompt_results] + mean_results[m] = sum(all_res)/len(all_res) + min_results[m] = min(all_res) + max_results[m] = max(all_res) + + print("======================================================") + print("Mean results: ", mean_results) + print("Min results: ", min_results) + print("Max results: ", max_results) + print("======================================================") + + save_data={} + save_data["test_prompt_ids"] = args.test_prompt_ids + save_data["mean_results"] = mean_results + save_data["min_results"] = min_results + save_data["max_results"] = max_results + save_data["all_prompt_results"] = all_prompt_results + + with open(args.results_file, "w") as f: + json.dump(save_data, f, indent = 4) + print("Save file: ", args.results_file) + + import smtplib + from email.mime.text import MIMEText + mail_host = 'smtp.qq.com' + mail_code = 'ouzplpngooqndjcb' + sender = '1849334588@qq.com' + receiver = 'esperanto1949@foxmail.com' + + task = '[v33: evaluate rqlora on arts]' + message = MIMEText('Task {task} Finished'.format(task = task), 'plain', 'utf-8') + message['Subject'] = 'Auto Email' + message['From'] = sender + message['To'] = receiver + + server = smtplib.SMTP_SSL("smtp.qq.com", 465) + server.login(sender, mail_code) + server.sendmail(sender, receiver, message.as_string()) + + server.quit() diff --git a/evaluate.py b/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..9a61f8bcaaa4a26a2e9438835dd403663b55e71e --- /dev/null +++ b/evaluate.py @@ -0,0 +1,69 @@ +import math + +def get_topk_results(predictions, scores, targets, k, all_items=None): + # target: [''] + results = [] + B = len(targets) + predictions = [_.split("Response:")[-1] for _ in predictions] + predictions = [_.strip().replace(" ","") for _ in predictions] + # prediction: ['', '', ''] + + if all_items is not None: + for i, seq in enumerate(predictions): + if seq not in all_items: + scores[i] = -1000 + + for b in range(B): + batch_seqs = predictions[b * k: (b + 1) * k] + batch_scores = scores[b * k: (b + 1) * k] + + pairs = [(a, b) for a, b in zip(batch_seqs, batch_scores)] + sorted_pairs = sorted(pairs, key=lambda x: x[1], reverse=True) + target_item = targets[b] + one_results = [] + for sorted_pred in sorted_pairs: + if sorted_pred[0] == target_item: + one_results.append(1) + else: + one_results.append(0) + + results.append(one_results) + + # result: [[0, 0, 0]] + return results + +def get_metrics_results(topk_results, metrics): + res = {} + for m in metrics: + if m.lower().startswith("hit"): + k = int(m.split("@")[1]) + res[m] = hit_k(topk_results, k) + elif m.lower().startswith("ndcg"): + k = int(m.split("@")[1]) + res[m] = ndcg_k(topk_results, k) + else: + raise NotImplementedError + + return res + + +def ndcg_k(topk_results, k): + + ndcg = 0.0 + for row in topk_results: + res = row[:k] + one_ndcg = 0.0 + for i in range(len(res)): + one_ndcg += res[i] / math.log(i + 2, 2) + ndcg += one_ndcg + return ndcg + + +def hit_k(topk_results, k): + hit = 0.0 + for row in topk_results: + res = row[:k] + if sum(res) > 0: + hit += 1 + return hit + diff --git a/fine-tune.py b/fine-tune.py new file mode 100644 index 0000000000000000000000000000000000000000..804ca0a3a07631bce2e2951ebede3619f365cd48 --- /dev/null +++ b/fine-tune.py @@ -0,0 +1,155 @@ +import argparse +import os +import sys +from typing import List + +import torch +import transformers +from peft import PeftModel +from peft import ( + TaskType, + LoraConfig, + get_peft_model, + get_peft_model_state_dict, + set_peft_model_state_dict, +) +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig + +from utils import * +from collator import Collator + +import argparse +from utils import * +from rq_llama import * + +parser = argparse.ArgumentParser(description = 'rqllama-finetune') +parser = parse_finetune_args(parser) +args = parser.parse_args() + +set_seed(args.seed) +ensure_dir(args.output_dir) + +device_map = "auto" +world_size = int(os.environ.get("WORLD_SIZE", 1)) +ddp = world_size != 1 +local_rank = int(os.environ.get("LOCAL_RANK") or 0) +if local_rank == 0: + print(vars(args)) + +if ddp: + device_map = {"": local_rank} + +train_data, valid_data = load_finetune_datasets(args) + +rqllama = LlamaWithRQ.from_pretrained(args.ckpt_path, torch_dtype = torch.float16, low_cpu_mem_usage = True, device_map = device_map) +tokenizer = rqllama.tokenizer +# PeftModelForCausalLM +model = rqllama.model +device = rqllama.device + +'''postfix = '' +new_tokens = [] +new_ids = list(range(args.re_index)) +for i in new_ids: + new_tokens.append(postfix.format(int(i))) +tokenizer.add_tokens(new_tokens) +''' + +if local_rank == 0: + print("token num:", len(rqllama.tokenizer)) + print("data num:", len(train_data)) + +collator = Collator(args, tokenizer) + +''' +# Re-index Embedding +new_ids = torch.tensor(new_ids, dtype = torch.float16).reshape(-1,1) +re_index_emb = torch.nn.Linear(1, model.config.hidden_size, dtype = torch.float16).to(device) +new_embeddings = re_index_emb(new_ids.to(device)) +# PeftModelForCausalLM -> LlamaForCausalLM -> LlamaModel +model.model.model.embed_tokens.original_module.weight.data = torch.cat([model.model.model.embed_tokens.original_module.weight.data, new_embeddings], dim = 0) +model.model.model.embed_tokens.modules_to_save.default.weight.data = torch.cat([model.model.model.embed_tokens.modules_to_save.default.weight.data, new_embeddings], dim = 0) + +new_lm_head = torch.randn(args.re_index, model.config.hidden_size, requires_grad = True).to(device) +# print('new_lm_head:',new_lm_head.requires_grad) +# PeftModelForCausalLM -> LlamaForCausalLM +model.model.lm_head.original_module.weight.data = torch.cat([model.model.lm_head.original_module.weight.data, new_lm_head], dim = 0) +model.model.lm_head.modules_to_save.default.weight.data = torch.cat([model.model.lm_head.modules_to_save.default.weight.data, new_lm_head], dim = 0) + +model.config.vocab_size = len(tokenizer) + +# print(model.model.model.embed_tokens.original_module.weight.shape) +# print(len(tokenizer)) +''' + +model.train() + +if local_rank == 0: + model.print_trainable_parameters() + +trainer = transformers.Trainer( + model = model, + train_dataset = train_data, + eval_dataset = valid_data, + args = transformers.TrainingArguments( + seed = args.seed, + per_device_train_batch_size = args.per_device_batch_size, + per_device_eval_batch_size = args.per_device_batch_size, + gradient_accumulation_steps = args.gradient_accumulation_steps, + warmup_ratio = args.warmup_ratio, + num_train_epochs = args.epochs, + learning_rate = args.learning_rate, + weight_decay = args.weight_decay, + lr_scheduler_type = args.lr_scheduler_type, + fp16 = args.fp16, + bf16 = args.bf16, + logging_steps = args.logging_step, + optim = args.optim, + gradient_checkpointing = True, + evaluation_strategy = args.save_and_eval_strategy, + save_strategy = args.save_and_eval_strategy, + eval_steps = args.save_and_eval_steps, + save_steps = args.save_and_eval_steps, + output_dir = args.output_dir, + save_total_limit = 50, + load_best_model_at_end = True, + deepspeed = args.deepspeed, + ddp_find_unused_parameters = False if ddp else None, + report_to = None, + eval_delay = 1 if args.save_and_eval_strategy=="epoch" else 2000, + dataloader_num_workers = args.dataloader_num_workers, + dataloader_prefetch_factor = args.dataloader_prefetch_factor, + remove_unused_columns = args.remove_unused_columns, + ), + tokenizer = tokenizer, + data_collator = collator, +) +model.config.use_cache = False + +if torch.__version__ >= "2" and sys.platform != "win32": + model = torch.compile(model) + +trainer.train(resume_from_checkpoint = args.resume_from_checkpoint) + +trainer.save_state() +trainer.save_model(output_dir = args.output_dir) + +if local_rank == 0: + import smtplib + from email.mime.text import MIMEText + mail_host = 'smtp.qq.com' + mail_code = 'ouzplpngooqndjcb' + sender = '1849334588@qq.com' + receiver = 'esperanto1949@foxmail.com' + + task = '[A100: finetune tt.llama]' + message = MIMEText('Task {task} Finished'.format(task = task), 'plain', 'utf-8') + message['Subject'] = 'Auto Email' + message['From'] = sender + message['To'] = receiver + + server = smtplib.SMTP_SSL("smtp.qq.com", 465) + server.login(sender, mail_code) + server.sendmail(sender, receiver, message.as_string()) + + server.quit() diff --git a/flops_profiler.out b/flops_profiler.out new file mode 100644 index 0000000000000000000000000000000000000000..9bdc3527f41ab92424aafaf694b180bb451b07a6 --- /dev/null +++ b/flops_profiler.out @@ -0,0 +1,4781 @@ + +-------------------------- DeepSpeed Flops Profiler -------------------------- +Profile Summary at step 11: +Notations: +data parallel size (dp_size), model parallel size(mp_size), +number of parameters (params), number of multiply-accumulate operations(MACs), +number of floating-point operations (flops), floating-point operations per second (FLOPS), +fwd latency (forward propagation latency), bwd latency (backward propagation latency), +step (weights update latency), iter latency (sum of fwd, bwd and step latency) + +world size: 4 +data parallel size: 4 +model parallel size: 1 +batch size per GPU: 8 +params per GPU: 7.04 B +params of model = params per GPU * mp_size: 7.04 B +fwd MACs per GPU: 28.95 TMACs +fwd flops per GPU: 57.9 T +fwd flops of model = fwd flops per GPU * mp_size: 57.9 T +fwd latency: 473.36 ms +fwd FLOPS per GPU = fwd flops per GPU / fwd latency: 122.32 TFLOPS +bwd latency: 883.27 ms +bwd FLOPS per GPU = 2 * fwd flops per GPU / bwd latency: 131.11 TFLOPS +fwd+bwd FLOPS per GPU = 3 * fwd flops per GPU / (fwd+bwd latency): 128.04 TFLOPS +step latency: 19.49 ms +iter latency: 1.38 s +FLOPS per GPU = 3 * fwd flops per GPU / iter latency: 126.23 TFLOPS +samples/second: 23.25 + +----------------------------- Aggregated Profile per GPU ----------------------------- +Top 3 modules in terms of params, MACs or fwd latency at different model depths: +depth 0: + params - {'PeftModelForCausalLM': '7.04 B'} + MACs - {'PeftModelForCausalLM': '28.95 TMACs'} + fwd latency - {'PeftModelForCausalLM': '470.45 ms'} +depth 1: + params - {'LoraModel': '7.04 B'} + MACs - {'LoraModel': '28.95 TMACs'} + fwd latency - {'LoraModel': '470.45 ms'} +depth 2: + params - {'LlamaForCausalLM': '7.04 B'} + MACs - {'LlamaForCausalLM': '28.95 TMACs'} + fwd latency - {'LlamaForCausalLM': '470.45 ms'} +depth 3: + params - {'LlamaModel': '6.77 B', 'ModulesToSaveWrapper': '270.53 M'} + MACs - {'LlamaModel': '28.38 TMACs', 'ModulesToSaveWrapper': '567.04 GMACs'} + fwd latency - {'LlamaModel': '466.15 ms', 'ModulesToSaveWrapper': '4.3 ms'} +depth 4: + params - {'ModuleList': '6.5 B', 'ModulesToSaveWrapper': '270.53 M', 'Linear': '135.27 M'} + MACs - {'ModuleList': '28.38 TMACs', 'ModuleDict': '567.04 GMACs', 'LlamaRotaryEmbedding': '67.07 KMACs'} + fwd latency - {'ModuleList': '457.32 ms', 'ModuleDict': '4.3 ms', 'LlamaRMSNorm': '518.56 us'} +depth 5: + params - {'LlamaDecoderLayer': '6.5 B', 'Embedding': '135.27 M', 'ModuleDict': '135.27 M'} + MACs - {'LlamaDecoderLayer': '28.38 TMACs', 'Linear': '567.04 GMACs', 'Embedding': '0 MACs'} + fwd latency - {'LlamaDecoderLayer': '457.32 ms', 'Linear': '4.3 ms', 'ModuleDict': '274.18 us'} +depth 6: + params - {'LlamaMLP': '4.34 B', 'LlamaSdpaAttention': '2.16 B', 'Embedding': '135.27 M'} + MACs - {'LlamaMLP': '18.19 TMACs', 'LlamaSdpaAttention': '10.19 TMACs', 'Embedding': '0 MACs'} + fwd latency - {'LlamaMLP': '214.52 ms', 'LlamaSdpaAttention': '202.41 ms', 'LlamaRMSNorm': '32.96 ms'} +depth 7: + params - {'Linear': '6.5 B', 'LlamaRotaryEmbedding': '0', 'SiLU': '0'} + MACs - {'Linear': '27.23 TMACs', 'LlamaRotaryEmbedding': '0 MACs', 'SiLU': '0 MACs'} + fwd latency - {'Linear': '344.65 ms', 'SiLU': '4.54 ms', 'LlamaRotaryEmbedding': '0 s'} +depth 8: + params - {'Linear': '6.48 B', 'ModuleDict': '19.99 M', 'ParameterDict': '0'} + MACs - {'Linear': '27.15 TMACs', 'ModuleDict': '83.79 GMACs', 'ParameterDict': '0 MACs'} + fwd latency - {'Linear': '224.12 ms', 'ModuleDict': '60.59 ms', 'ParameterDict': '0 s'} + +------------------------------ Detailed Profile per GPU ------------------------------ +Each module profile is listed after its name in the following order: +params, percentage of total params, MACs, percentage of total MACs, fwd latency, percentage of total fwd latency, fwd FLOPS + +Note: 1. A module can have torch.nn.module or torch.nn.functional to compute logits (e.g. CrossEntropyLoss). They are not counted as submodules, thus not to be printed out. However they make up the difference between a parent's MACs (or latency) and the sum of its submodules'. +2. Number of floating-point operations is a theoretical estimation, thus FLOPS computed using that could be larger than the maximum system throughput. +3. The fwd latency listed in the top module's profile is directly captured at the module forward function in PyTorch, thus it's less than the fwd latency shown above which is captured in DeepSpeed. + +PeftModelForCausalLM( + 7.04 B = 100% Params, 28.95 TMACs = 100% MACs, 470.45 ms = 100% latency, 123.08 TFLOPS + (base_model): LoraModel( + 7.04 B = 100% Params, 28.95 TMACs = 100% MACs, 470.45 ms = 100% latency, 123.08 TFLOPS + (model): LlamaForCausalLM( + 7.04 B = 100% Params, 28.95 TMACs = 100% MACs, 470.45 ms = 100% latency, 123.08 TFLOPS + (model): LlamaModel( + 6.77 B = 96.16% Params, 28.38 TMACs = 98.04% MACs, 466.15 ms = 99.09% latency, 121.78 TFLOPS + (embed_tokens): ModulesToSaveWrapper( + 270.53 M = 3.84% Params, 0 MACs = 0% MACs, 274.18 us = 0.06% latency, 0 FLOPS + (original_module): Embedding(135.27 M = 1.92% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS, 33024, 4096, padding_idx=0) + (modules_to_save): ModuleDict( + 135.27 M = 1.92% Params, 0 MACs = 0% MACs, 274.18 us = 0.06% latency, 0 FLOPS + (default): Embedding(135.27 M = 1.92% Params, 0 MACs = 0% MACs, 274.18 us = 0.06% latency, 0 FLOPS, 33024, 4096, padding_idx=0) + ) + ) + (layers): ModuleList( + (0): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.39 ms = 3.06% latency, 123.25 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.42 ms = 1.37% latency, 99.13 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.26 ms = 0.27% latency, 112 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 720.02 us = 0.15% latency, 195.35 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 111.58 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 111.58 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 96.56 us = 0.02% latency, 2.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 96.56 us = 0.02% latency, 2.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 98.71 us = 0.02% latency, 2.78 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 98.71 us = 0.02% latency, 2.78 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.28 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 650.17 us = 0.14% latency, 216.34 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.09 ms = 0.23% latency, 128.98 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 649.21 us = 0.14% latency, 216.66 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.47 us = 0.01% latency, 4.07 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.47 us = 0.01% latency, 4.07 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.2 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 653.98 us = 0.14% latency, 215.08 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.19 us = 0.02% latency, 3.75 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.19 us = 0.02% latency, 3.75 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.71 ms = 1.43% latency, 169.43 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.55 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.91 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 123.5 us = 0.03% latency, 5.98 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 123.5 us = 0.03% latency, 5.98 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.33 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.47 ms = 0.31% latency, 257.27 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 78.68 us = 0.02% latency, 3.49 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 78.68 us = 0.02% latency, 3.49 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.71 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.31 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.22 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.22 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 124.45 us = 0.03% latency, 5.93 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 124.45 us = 0.03% latency, 5.93 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 149.25 us = 0.03% latency, 309.18 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.12 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.08 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (1): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.2 ms = 3.02% latency, 124.96 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.28 ms = 1.33% latency, 101.42 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.88 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.8 us = 0.14% latency, 213.84 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.25 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 652.31 us = 0.14% latency, 215.63 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.48 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 652.07 us = 0.14% latency, 215.71 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 66.76 us = 0.01% latency, 4.12 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 66.76 us = 0.01% latency, 4.12 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.88 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.8 us = 0.14% latency, 213.84 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.65 ms = 1.41% latency, 170.97 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.96 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.86 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 102.76 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 102.76 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.13 ms = 0.45% latency, 177.97 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.47 ms = 0.31% latency, 256.6 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.98 ms = 0.42% latency, 191.84 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 272.38 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.5 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.5 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.47 us = 0.01% latency, 4.07 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.47 us = 0.01% latency, 4.07 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 144 us = 0.03% latency, 320.44 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.32 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 515.22 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (2): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.26 ms = 3.03% latency, 124.43 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.32 ms = 1.34% latency, 100.69 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.15 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 665.9 us = 0.14% latency, 211.23 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.18 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.47 us = 0.14% latency, 213.29 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.06 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 653.03 us = 0.14% latency, 215.4 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 65.8 us = 0.01% latency, 4.17 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 65.8 us = 0.01% latency, 4.17 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.14 ms = 0.24% latency, 124.35 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 79.63 us = 0.02% latency, 3.45 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 79.63 us = 0.02% latency, 3.45 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.68 ms = 1.42% latency, 170.31 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 177.04 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.86 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.1 us = 0.02% latency, 6.65 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.1 us = 0.02% latency, 6.65 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.31 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 256.27 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.03 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 272.01 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 203.37 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 203.37 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.25 us = 0.02% latency, 6.52 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.25 us = 0.02% latency, 6.52 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 142.34 us = 0.03% latency, 324.2 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.36 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.79 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (3): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.26 ms = 3.03% latency, 124.39 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.31 ms = 1.34% latency, 100.99 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.88 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 660.66 us = 0.14% latency, 212.91 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 99.18 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 99.18 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.67 us = 0.02% latency, 3.73 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.67 us = 0.02% latency, 3.73 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.09 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 652.79 us = 0.14% latency, 215.48 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.48 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.89 us = 0.14% latency, 214.46 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 66.52 us = 0.01% latency, 4.13 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 66.52 us = 0.01% latency, 4.13 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.46 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.65 us = 0.14% latency, 214.54 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 169.89 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 175.92 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.83 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 78.68 us = 0.02% latency, 3.49 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 78.68 us = 0.02% latency, 3.49 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.37 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 256.11 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.87 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.78 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.22 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.22 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 140.91 us = 0.03% latency, 327.49 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.88 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 516.41 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (4): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.28 ms = 3.04% latency, 124.21 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.6 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.38 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.47 us = 0.14% latency, 213.29 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.99 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 654.94 us = 0.14% latency, 214.77 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 75.58 us = 0.02% latency, 3.63 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 75.58 us = 0.02% latency, 3.63 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.04 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 654.46 us = 0.14% latency, 214.93 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 80.59 us = 0.02% latency, 3.41 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 80.59 us = 0.02% latency, 3.41 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.08 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.04 us = 0.14% latency, 213.76 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 169.93 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.49 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.2 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 177.06 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.65 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.49 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.78 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 197.17 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 197.17 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 144 us = 0.03% latency, 320.44 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.88 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 517.85 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (5): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.25 ms = 3.03% latency, 124.45 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.31 ms = 1.34% latency, 100.85 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.54 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 660.18 us = 0.14% latency, 213.06 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 99.42 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 99.42 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.37 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 654.94 us = 0.14% latency, 214.77 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 127.87 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 653.27 us = 0.14% latency, 215.32 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.38 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.04 us = 0.14% latency, 213.76 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.68 ms = 1.42% latency, 170.19 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.12 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.83 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.23 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.53 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.2 us = 0.02% latency, 6.76 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.2 us = 0.02% latency, 6.76 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.3 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.48 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.98 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.98 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 138.52 us = 0.03% latency, 333.13 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.88 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.55 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (6): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.25 ms = 3.03% latency, 124.47 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.34% latency, 100.67 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.94 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.7 us = 0.14% latency, 213.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.02 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.57 us = 0.14% latency, 212.3 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.66 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.65 us = 0.14% latency, 214.54 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.16 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.56 us = 0.14% latency, 213.91 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.67 ms = 1.42% latency, 170.52 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.98 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.24 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.12 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.45 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.98 ms = 0.42% latency, 191.01 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.73 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.03 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.03 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.34 us = 0.02% latency, 6.63 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.34 us = 0.02% latency, 6.63 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 140.67 us = 0.03% latency, 328.05 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.88 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 516.41 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (7): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.27 ms = 3.03% latency, 124.34 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.32 ms = 1.34% latency, 100.73 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 124.74 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.09 us = 0.14% latency, 212.45 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 104.67 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 104.67 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.67 us = 0.02% latency, 3.73 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.67 us = 0.02% latency, 3.73 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.23 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.17 us = 0.14% latency, 214.69 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.23% latency, 127.78 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 654.46 us = 0.14% latency, 214.93 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.24 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.68 ms = 1.42% latency, 170.15 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.43 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.87 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.92 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.45 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.26 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.68 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 117.06 us = 0.02% latency, 6.31 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 117.06 us = 0.02% latency, 6.31 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 139.95 us = 0.03% latency, 329.72 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.17 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.08 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (8): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.27 ms = 3.03% latency, 124.33 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.31 ms = 1.34% latency, 100.85 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.94 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 660.42 us = 0.14% latency, 212.99 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.4 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 654.46 us = 0.14% latency, 214.93 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.96 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.04 us = 0.14% latency, 213.76 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.51 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.08 us = 0.14% latency, 214.07 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 169.86 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.47 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.26 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.53 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.54 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.99 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 270.99 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 201.46 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 201.46 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.34 us = 0.02% latency, 6.63 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.34 us = 0.02% latency, 6.63 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 140.19 us = 0.03% latency, 329.16 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.17 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 516.18 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (9): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.27 ms = 3.03% latency, 124.35 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.32 ms = 1.34% latency, 100.78 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.11 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 663.04 us = 0.14% latency, 212.14 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.91 us = 0.02% latency, 3.72 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.91 us = 0.02% latency, 3.72 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.23% latency, 127.78 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.41 us = 0.14% latency, 214.61 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.32 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.84 us = 0.14% latency, 214.15 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.54 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.8 us = 0.14% latency, 213.84 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.68 ms = 1.42% latency, 170.23 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.47 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.3 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.62 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.54 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.74 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.45 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 197.17 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 197.17 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 139.71 us = 0.03% latency, 330.29 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.55 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 515.94 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (10): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.29 ms = 3.04% latency, 124.11 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.56 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.8 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.7 us = 0.14% latency, 213.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.83 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.13 us = 0.14% latency, 214.38 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.64 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.32 us = 0.14% latency, 213.99 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.62 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.84 us = 0.14% latency, 214.15 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.7 ms = 1.42% latency, 169.74 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.59 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.83 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 177.02 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 255.08 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.43% latency, 189.4 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.71 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.03 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.03 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 119.21 us = 0.03% latency, 6.19 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 119.21 us = 0.03% latency, 6.19 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 141.14 us = 0.03% latency, 326.94 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.88 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 517.37 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (11): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.39 ms = 3.06% latency, 123.25 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.32 ms = 1.34% latency, 100.77 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.54 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 663.76 us = 0.14% latency, 211.92 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.02 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.51 us = 0.14% latency, 213.6 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.74 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.33 us = 0.14% latency, 212.37 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.67 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.08 us = 0.14% latency, 214.07 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.63 us = 0.02% latency, 3.68 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.63 us = 0.02% latency, 3.68 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.81 ms = 1.45% latency, 167.01 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.19 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.58 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.73 us = 0.02% latency, 6.49 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.73 us = 0.02% latency, 6.49 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.72 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.26 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.01 ms = 0.43% latency, 188.57 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.48 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 198.13 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 198.13 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 117.06 us = 0.02% latency, 6.31 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 117.06 us = 0.02% latency, 6.31 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 141.14 us = 0.03% latency, 326.94 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.32 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.56 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (12): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.29 ms = 3.04% latency, 124.18 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.34% latency, 100.67 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.54 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 661.13 us = 0.14% latency, 212.76 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.85 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.8 us = 0.14% latency, 212.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.74 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.65 us = 0.14% latency, 214.54 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.3 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.75 us = 0.14% latency, 213.53 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 169.87 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.1 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.09 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.94 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.95 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.99 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.13 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.26 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.26 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 115.63 us = 0.02% latency, 6.39 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 115.63 us = 0.02% latency, 6.39 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 141.14 us = 0.03% latency, 326.94 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 514.27 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 514.51 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (13): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.29 ms = 3.04% latency, 124.15 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.34% latency, 100.64 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 124.58 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 664.95 us = 0.14% latency, 211.54 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.85 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.17 us = 0.14% latency, 214.69 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.06 us = 0.02% latency, 3.61 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.06 us = 0.02% latency, 3.61 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.54 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.13 us = 0.14% latency, 214.38 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.51 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.51 us = 0.14% latency, 213.6 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.7 ms = 1.42% latency, 169.8 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.17 ms = 0.46% latency, 174.93 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.44 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.01 us = 0.02% latency, 6.53 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.01 us = 0.02% latency, 6.53 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 177 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.79 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.71 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.22 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.26 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.26 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 139.24 us = 0.03% latency, 331.42 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.36 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.32 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (14): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.25 ms = 3.03% latency, 124.47 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.3 ms = 1.34% latency, 101.09 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.28 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 661.13 us = 0.14% latency, 212.76 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 127.81 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.41 us = 0.14% latency, 214.61 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.09 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.41 us = 0.14% latency, 214.61 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.54 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.51 us = 0.14% latency, 213.6 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 170.08 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.57 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.5 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.39 us = 0.02% latency, 6.69 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.39 us = 0.02% latency, 6.69 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.7 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.99 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.53 us = 0.02% latency, 6.56 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.53 us = 0.02% latency, 6.56 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.44 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 271.08 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 197.17 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 197.17 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 140.91 us = 0.03% latency, 327.49 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.36 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 520.71 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (15): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.24 ms = 3.03% latency, 124.57 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.29 ms = 1.34% latency, 101.17 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.28 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 661.61 us = 0.14% latency, 212.6 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.06 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 654.7 us = 0.14% latency, 214.85 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.06 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 654.94 us = 0.14% latency, 214.77 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.75 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.8 us = 0.14% latency, 213.84 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.68 ms = 1.42% latency, 170.19 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.06 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.85 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.39 us = 0.02% latency, 6.69 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.39 us = 0.02% latency, 6.69 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.25 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 255.03 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.28 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.25 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 197.41 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 197.41 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 139 us = 0.03% latency, 331.99 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.84 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 519.99 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (16): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.46 ms = 3.07% latency, 122.7 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.59 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 124.98 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 661.85 us = 0.14% latency, 212.53 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.86 us = 0.02% latency, 3.67 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.86 us = 0.02% latency, 3.67 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.62 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.89 us = 0.14% latency, 214.46 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.67 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.89 us = 0.14% latency, 214.46 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.78 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.8 us = 0.14% latency, 213.84 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.85 ms = 1.46% latency, 166.12 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.24 ms = 0.48% latency, 169.22 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.5 ms = 0.32% latency, 251.99 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 109.91 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 109.91 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 81.06 us = 0.02% latency, 3.39 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 81.06 us = 0.02% latency, 3.39 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 123.74 us = 0.03% latency, 5.97 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 123.74 us = 0.03% latency, 5.97 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.18 ms = 0.46% latency, 173.82 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.36 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 99.9 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 99.9 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 108.96 us = 0.02% latency, 6.78 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 108.96 us = 0.02% latency, 6.78 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.85 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.94 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 197.41 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 197.41 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 147.1 us = 0.03% latency, 313.69 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.84 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 521.18 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (17): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.27 ms = 3.03% latency, 124.34 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.3 ms = 1.34% latency, 101.03 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.96 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.8 us = 0.14% latency, 212.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 127.87 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.37 us = 0.14% latency, 214.3 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.23% latency, 127.78 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.32 us = 0.14% latency, 213.99 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.4 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.47 us = 0.14% latency, 213.29 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.7 ms = 1.43% latency, 169.62 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.04 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.85 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.27 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.31% latency, 255.16 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 93.94 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 93.94 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 108.96 us = 0.02% latency, 6.78 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 108.96 us = 0.02% latency, 6.78 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.01 ms = 0.43% latency, 188.5 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.42 ms = 0.3% latency, 267.15 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 194.79 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 194.79 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 115.63 us = 0.02% latency, 6.39 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 115.63 us = 0.02% latency, 6.39 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 142.34 us = 0.03% latency, 324.2 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.12 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 517.85 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (18): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.29 ms = 3.04% latency, 124.13 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.57 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.35 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 661.37 us = 0.14% latency, 212.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.53 us = 0.02% latency, 3.59 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.53 us = 0.02% latency, 3.59 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.64 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.6 us = 0.14% latency, 214.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.51 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.65 us = 0.14% latency, 214.54 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 93.46 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 93.46 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.91 us = 0.02% latency, 3.72 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.91 us = 0.02% latency, 3.72 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 124.48 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.99 us = 0.14% latency, 213.45 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.7 ms = 1.42% latency, 169.72 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.88 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.77 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.63 us = 0.02% latency, 6.67 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.33 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.71 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.17 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 270.99 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.74 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.74 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.96 us = 0.02% latency, 6.48 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.96 us = 0.02% latency, 6.48 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 140.43 us = 0.03% latency, 328.61 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.88 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 516.65 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (19): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.26 ms = 3.03% latency, 124.43 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.31 ms = 1.34% latency, 100.91 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.83 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 663.04 us = 0.14% latency, 212.14 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.02 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.89 us = 0.14% latency, 214.46 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.39 us = 0.02% latency, 3.69 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.39 us = 0.02% latency, 3.69 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 128.03 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.65 us = 0.14% latency, 214.54 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.24 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.94 us = 0.14% latency, 213.14 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.68 ms = 1.42% latency, 170.13 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.45 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.53 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.45% latency, 177.33 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.91 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.2 us = 0.02% latency, 6.76 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.2 us = 0.02% latency, 6.76 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.76 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.39 ms = 0.3% latency, 270.99 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.46 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.46 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.25 us = 0.02% latency, 6.52 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.25 us = 0.02% latency, 6.52 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 139.95 us = 0.03% latency, 329.72 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.65 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.8 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (20): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.28 ms = 3.04% latency, 124.2 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.53 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.83 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 661.85 us = 0.14% latency, 212.53 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.96 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 661.85 us = 0.14% latency, 212.53 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.96 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.6 us = 0.14% latency, 214.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.86 us = 0.02% latency, 3.67 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.86 us = 0.02% latency, 3.67 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.14 ms = 0.24% latency, 123.98 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 668.53 us = 0.14% latency, 210.4 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 170.04 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.53 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.89 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 177.04 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.5 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.18 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.18 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.2 us = 0.02% latency, 6.76 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.2 us = 0.02% latency, 6.76 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.87 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.34 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 198.36 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 198.36 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.53 us = 0.02% latency, 6.56 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.53 us = 0.02% latency, 6.56 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 141.62 us = 0.03% latency, 325.84 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.88 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.56 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (21): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.32 ms = 3.04% latency, 123.84 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.34 ms = 1.35% latency, 100.45 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.08 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 664.47 us = 0.14% latency, 211.69 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.88 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.23 us = 0.14% latency, 213.37 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.71 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.56 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 124.98 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.09 us = 0.14% latency, 212.45 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.9 us = 0.01% latency, 3.99 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.72 ms = 1.43% latency, 169.28 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.17 ms = 0.46% latency, 174.65 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 115.39 us = 0.02% latency, 6.4 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 115.39 us = 0.02% latency, 6.4 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.23 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.22 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.89 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.25 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 198.6 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 198.6 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.77 us = 0.02% latency, 6.55 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.77 us = 0.02% latency, 6.55 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 139.24 us = 0.03% latency, 331.42 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 515.22 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.08 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (22): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.35 ms = 3.05% latency, 123.65 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.37 ms = 1.35% latency, 99.92 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.03 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 667.81 us = 0.14% latency, 210.63 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.19 us = 0.02% latency, 3.75 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.19 us = 0.02% latency, 3.75 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.66 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 666.38 us = 0.14% latency, 211.08 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.37 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.23 us = 0.14% latency, 213.37 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.14 ms = 0.24% latency, 123.55 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 663.52 us = 0.14% latency, 211.99 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.94 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.06 us = 0.02% latency, 3.61 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.06 us = 0.02% latency, 3.61 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.71 ms = 1.43% latency, 169.5 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.61 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.5 ms = 0.32% latency, 252.84 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.62 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.22 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.99 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.57 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 199.08 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 199.08 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.53 us = 0.02% latency, 6.56 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.53 us = 0.02% latency, 6.56 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 149.73 us = 0.03% latency, 308.2 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 514.27 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 520.94 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (23): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.35 ms = 3.05% latency, 123.62 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.37 ms = 1.35% latency, 100.02 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.18 ms = 0.25% latency, 119.72 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 665.66 us = 0.14% latency, 211.31 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 99.18 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 99.18 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.23% latency, 127.76 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.84 us = 0.14% latency, 214.15 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.43 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.84 us = 0.14% latency, 214.15 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 74.15 us = 0.02% latency, 3.71 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.43 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.94 us = 0.14% latency, 213.14 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.23 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.72 ms = 1.43% latency, 169.21 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.3 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.4 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.91 us = 0.02% latency, 6.72 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.14 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.42 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 114.68 us = 0.02% latency, 6.44 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 114.68 us = 0.02% latency, 6.44 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.43% latency, 189.31 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.53 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 114.2 us = 0.02% latency, 6.47 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 114.2 us = 0.02% latency, 6.47 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 140.43 us = 0.03% latency, 328.61 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.08 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 515.94 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (24): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.28 ms = 3.03% latency, 124.25 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.55 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.64 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.8 us = 0.14% latency, 212.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 98.47 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.72 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.67 us = 0.02% latency, 3.73 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.67 us = 0.02% latency, 3.73 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.83 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.6 us = 0.14% latency, 214.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 100.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 100.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 124.9 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.7 us = 0.14% latency, 213.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.29 us = 0.02% latency, 3.6 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.29 us = 0.02% latency, 3.6 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 170 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.47 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.05 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.37 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 108.72 us = 0.02% latency, 6.79 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 108.72 us = 0.02% latency, 6.79 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.66 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.54 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.18 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.18 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.39 us = 0.02% latency, 6.69 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.39 us = 0.02% latency, 6.69 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.05 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.66 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.01 us = 0.02% latency, 6.53 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.01 us = 0.02% latency, 6.53 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 139.95 us = 0.03% latency, 329.72 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 511.17 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 514.27 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (25): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.29 ms = 3.04% latency, 124.15 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.31 ms = 1.34% latency, 100.9 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.64 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 664.95 us = 0.14% latency, 211.54 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.45 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.6 us = 0.14% latency, 214.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 127.92 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 655.89 us = 0.14% latency, 214.46 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.16 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.8 us = 0.14% latency, 213.84 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 75.58 us = 0.02% latency, 3.63 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 75.58 us = 0.02% latency, 3.63 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.71 ms = 1.43% latency, 169.48 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.4 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.2 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.53 us = 0.02% latency, 3.59 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 76.53 us = 0.02% latency, 3.59 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 177.06 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.48 ms = 0.32% latency, 254.58 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 93.7 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 93.7 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.62 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.34 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.74 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.74 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 143.77 us = 0.03% latency, 320.98 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.36 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 516.89 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (26): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.29 ms = 3.04% latency, 124.14 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.32 ms = 1.34% latency, 100.82 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.14 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 663.04 us = 0.14% latency, 212.14 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 78.2 us = 0.02% latency, 3.51 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 78.2 us = 0.02% latency, 3.51 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.43 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.99 us = 0.14% latency, 213.45 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.07 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.32 us = 0.14% latency, 213.99 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 77.25 us = 0.02% latency, 3.56 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 77.25 us = 0.02% latency, 3.56 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.96 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.71 ms = 1.43% latency, 169.42 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 175.9 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.28 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.15 us = 0.02% latency, 6.7 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.06 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.3 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 113.49 us = 0.02% latency, 6.51 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.14 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.48 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.5 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.5 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 141.86 us = 0.03% latency, 325.29 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.36 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.56 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (27): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.28 ms = 3.04% latency, 124.2 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.32 ms = 1.34% latency, 100.79 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.59 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 664.23 us = 0.14% latency, 211.76 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.72 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.56 us = 0.14% latency, 213.91 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.51 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.32 us = 0.14% latency, 213.99 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.46 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.23 us = 0.14% latency, 213.37 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.7 ms = 1.42% latency, 169.69 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.57 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.24 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 77.72 us = 0.02% latency, 3.53 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 77.72 us = 0.02% latency, 3.53 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 110.86 us = 0.02% latency, 6.66 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.86 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.46 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 1.99 ms = 0.42% latency, 190.12 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.48 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 194.31 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 194.31 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.1 us = 0.02% latency, 6.65 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.1 us = 0.02% latency, 6.65 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.71 us = 0.01% latency, 4.06 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 140.67 us = 0.03% latency, 328.05 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.32 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.56 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (28): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.3 ms = 3.04% latency, 124.08 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.57 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.8 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 662.8 us = 0.14% latency, 212.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.99 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.48 us = 0.02% latency, 3.79 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 126.18 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.99 us = 0.14% latency, 213.45 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.8 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.43 us = 0.02% latency, 3.74 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 73.43 us = 0.02% latency, 3.74 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.66 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.84 us = 0.14% latency, 214.15 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.65 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 77.72 us = 0.02% latency, 3.53 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 77.72 us = 0.02% latency, 3.53 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.78 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 659.23 us = 0.14% latency, 213.37 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.1 us = 0.01% latency, 3.92 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.71 ms = 1.43% latency, 169.6 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 175.96 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.32 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.08 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.74 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.5 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 93.94 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 93.94 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.43 us = 0.02% latency, 6.75 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.69 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.66 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 197.41 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 197.41 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.06 us = 0.02% latency, 6.59 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 142.34 us = 0.03% latency, 324.2 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.6 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 516.65 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (29): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.26 ms = 3.03% latency, 124.43 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.31 ms = 1.34% latency, 100.98 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.43 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 664.71 us = 0.14% latency, 211.61 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.04 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.24 us = 0.02% latency, 3.8 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.65 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.8 us = 0.14% latency, 213.84 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.1 ms = 0.23% latency, 127.95 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.37 us = 0.14% latency, 214.3 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.61 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 67.95 us = 0.01% latency, 4.04 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.51 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.75 us = 0.14% latency, 213.53 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72 us = 0.02% latency, 3.82 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.69 ms = 1.42% latency, 169.97 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.88 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.48 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.84 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.14 ms = 0.46% latency, 176.96 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.38 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 109.67 us = 0.02% latency, 6.73 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.8 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 269.47 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 195.74 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 195.74 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 138.76 us = 0.03% latency, 332.56 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 512.36 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 517.85 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (30): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.33 ms = 3.05% latency, 123.78 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.33 ms = 1.35% latency, 100.58 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.56 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 665.9 us = 0.14% latency, 211.23 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.75 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.72 us = 0.02% latency, 3.78 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.38 us = 0.01% latency, 3.96 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.77 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 126.85 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.6 us = 0.14% latency, 214.22 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.14 us = 0.01% latency, 3.97 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.57 us = 0.02% latency, 3.89 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 124.58 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.51 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.74 ms = 1.43% latency, 168.79 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.18 ms = 0.46% latency, 173.84 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.16 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.53 us = 0.02% latency, 3.84 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 133.51 us = 0.03% latency, 5.53 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 133.51 us = 0.03% latency, 5.53 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.19 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.26 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 116.11 us = 0.02% latency, 6.36 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 116.11 us = 0.02% latency, 6.36 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.96 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 270.71 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.7 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 112.3 us = 0.02% latency, 6.57 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 148.06 us = 0.03% latency, 311.67 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 514.27 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 517.61 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + (31): LlamaDecoderLayer( + 203.01 M = 2.88% Params, 886.96 GMACs = 3.06% MACs, 14.27 ms = 3.03% latency, 124.3 TFLOPS + (self_attn): LlamaSdpaAttention( + 67.37 M = 0.96% Params, 318.41 GMACs = 1.1% MACs, 6.31 ms = 1.34% latency, 100.93 TFLOPS + (q_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.13 ms = 0.24% latency, 125.38 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 666.86 us = 0.14% latency, 210.93 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.56 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 72.96 us = 0.02% latency, 3.77 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.86 us = 0.01% latency, 3.93 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (k_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.24% latency, 127.62 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 656.84 us = 0.14% latency, 214.15 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 95.13 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.33 us = 0.01% latency, 3.91 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.66 us = 0.01% latency, 4 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (v_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.11 ms = 0.23% latency, 127.73 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 657.32 us = 0.14% latency, 213.99 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.89 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.05 us = 0.02% latency, 3.87 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.43 us = 0.01% latency, 4.01 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (o_proj): lora.Linear( + 16.84 M = 0.24% Params, 70.6 GMACs = 0.24% MACs, 1.12 ms = 0.24% latency, 125.7 TFLOPS + (base_layer): Linear(16.78 M = 0.24% Params, 70.33 GMACs = 0.24% MACs, 658.27 us = 0.14% latency, 213.68 TFLOPS, in_features=4096, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 97.27 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.76 us = 0.02% latency, 3.83 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 69.62 us = 0.01% latency, 3.95 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (mlp): LlamaMLP( + 135.63 M = 1.93% Params, 568.56 GMACs = 1.96% MACs, 6.7 ms = 1.42% latency, 169.7 TFLOPS + (gate_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.16 ms = 0.46% latency, 175.71 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 253.4 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 96.32 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 70.81 us = 0.02% latency, 3.88 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.82 us = 0.02% latency, 6.6 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (up_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2.15 ms = 0.46% latency, 176.53 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.49 ms = 0.32% latency, 254.54 TFLOPS, in_features=4096, out_features=11008, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 94.41 us = 0.02% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 71.29 us = 0.02% latency, 3.85 TFLOPS, in_features=4096, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.34 us = 0.02% latency, 6.63 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.34 us = 0.02% latency, 6.63 TFLOPS, in_features=8, out_features=11008, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (down_proj): lora.Linear( + 45.21 M = 0.64% Params, 189.52 GMACs = 0.65% MACs, 2 ms = 0.42% latency, 189.69 TFLOPS + (base_layer): Linear(45.09 M = 0.64% Params, 189.01 GMACs = 0.65% MACs, 1.4 ms = 0.3% latency, 269.51 TFLOPS, in_features=11008, out_features=4096, bias=False) + (lora_dropout): ModuleDict( + 0 = 0% Params, 0 MACs = 0% MACs, 196.93 us = 0.04% latency, 0 FLOPS + (default): Dropout(0 = 0% Params, 0 MACs = 0% MACs, 196.93 us = 0.04% latency, 0 FLOPS, p=0.05, inplace=False) + ) + (lora_A): ModuleDict( + 88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS + (default): Linear(88.06 K = 0% Params, 369.16 MMACs = 0% MACs, 111.58 us = 0.02% latency, 6.62 TFLOPS, in_features=11008, out_features=8, bias=False) + ) + (lora_B): ModuleDict( + 32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS + (default): Linear(32.77 K = 0% Params, 137.36 MMACs = 0% MACs, 68.19 us = 0.01% latency, 4.03 TFLOPS, in_features=8, out_features=4096, bias=False) + ) + (lora_embedding_A): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_embedding_B): ParameterDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + (lora_magnitude_vector): ModuleDict(0 = 0% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS) + ) + (act_fn): SiLU(0 = 0% Params, 0 MACs = 0% MACs, 138.76 us = 0.03% latency, 332.56 GFLOPS) + ) + (input_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 513.32 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (post_attention_layernorm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 517.85 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + ) + ) + (norm): LlamaRMSNorm(4.1 K = 0% Params, 0 MACs = 0% MACs, 518.56 us = 0.11% latency, 0 FLOPS, (4096,), eps=1e-06) + (rotary_emb): LlamaRotaryEmbedding(0 = 0% Params, 67.07 KMACs = 0% MACs, 411.03 us = 0.09% latency, 326.36 MFLOPS) + ) + (lm_head): ModulesToSaveWrapper( + 270.53 M = 3.84% Params, 567.04 GMACs = 1.96% MACs, 4.3 ms = 0.91% latency, 263.92 TFLOPS + (original_module): Linear(135.27 M = 1.92% Params, 0 MACs = 0% MACs, 0 s = 0% latency, 0 FLOPS, in_features=4096, out_features=33024, bias=False) + (modules_to_save): ModuleDict( + 135.27 M = 1.92% Params, 567.04 GMACs = 1.96% MACs, 4.3 ms = 0.91% latency, 263.92 TFLOPS + (default): Linear(135.27 M = 1.92% Params, 567.04 GMACs = 1.96% MACs, 4.3 ms = 0.91% latency, 263.92 TFLOPS, in_features=4096, out_features=33024, bias=False) + ) + ) + ) + ) +) +------------------------------------------------------------------------------ diff --git a/generate_embeddings.py b/generate_embeddings.py new file mode 100644 index 0000000000000000000000000000000000000000..915812862562e4026b166013a2d905a33bb389c4 --- /dev/null +++ b/generate_embeddings.py @@ -0,0 +1,57 @@ +import os +import collections +import json +import logging +import argparse +import numpy as np +import pandas as pd +import torch +from time import time +from torch import optim +from tqdm import tqdm +from torch.utils.data import DataLoader +from rq_llama import * + +def parse_args(): + parser = argparse.ArgumentParser(description = "Index") + parser.add_argument("--ckpt_path", type = str, default = "", help = "") + parser.add_argument("--save_path", type = str, default = "", help = "") + parser.add_argument("--device_map", type = str, default = "1", help = "gpu or cpu") + return parser.parse_args() + +args = parse_args() +print(args) +device_map = {'': int(args.device_map)} +MODEL = LlamaWithRQ.from_pretrained(args.ckpt_path, torch_dtype = torch.float16, low_cpu_mem_usage = True, device_map = device_map) +MODEL.eval() +device = MODEL.device +llama = MODEL.model.get_decoder() +tokenizer = MODEL.tokenizer +item_texts = MODEL.item_texts + +all_idx = [] +all_embeddings = [] +with torch.no_grad(): + for idx, text in tqdm(item_texts.items()): + item_text = text['title'] + ' ' + text['description'] + item_ids = tokenizer(item_text, return_tensors = 'pt', padding = True, truncation = True).to(device) + item_emb = llama(input_ids = item_ids.input_ids, attention_mask = item_ids.attention_mask) + item_emb = item_emb.last_hidden_state * item_ids.attention_mask.unsqueeze(-1) + item_emb = item_emb.sum(dim = 1) / item_ids.attention_mask.sum(dim = -1, keepdim = True) + + all_idx.append(idx) + all_embeddings.append(item_emb.detach().cpu().numpy().flatten().tolist()) + +results = { + 'id': all_idx, + 'emb': [] +} + +for emb in tqdm(all_embeddings): + str_emb = '' + for e in emb: + str_emb = str_emb + str(e) + ' ' + results['emb'].append(str_emb[:-1]) + +df = pd.DataFrame(results) +df.to_csv(args.save_path, sep = '\t', header = 0, index = False) \ No newline at end of file diff --git a/generate_indices.py b/generate_indices.py new file mode 100644 index 0000000000000000000000000000000000000000..f257e223dc4ceccdcf2d589ab41bad3429696d6d --- /dev/null +++ b/generate_indices.py @@ -0,0 +1,115 @@ +import os +import collections +import json +import logging +import argparse +import numpy as np +import pandas as pd +import torch +from time import time +from torch import optim +from tqdm import tqdm +from torch.utils.data import DataLoader + +from rq_llama import * +from index.datasets import EmbDataset + +def if_collided(all_indices_str): + tot_item = len(all_indices_str) + tot_indice = len(set(all_indices_str.tolist())) + return tot_item == tot_indice + +def get_indices_count(all_indices_str): + indices_count = collections.defaultdict(int) + for index in all_indices_str: + indices_count[index] += 1 + return indices_count + +def get_collision_item(all_indices_str): + index2id = {} + for i, index in enumerate(all_indices_str): + if index not in index2id: + index2id[index] = [] + index2id[index].append(i) + collision_item_groups = [] + for index in index2id: + if len(index2id[index]) > 1: + collision_item_groups.append(index2id[index]) + return collision_item_groups + +def parse_args(): + parser = argparse.ArgumentParser(description = "Index") + parser.add_argument("--ckpt_path", type = str, default = "", help = "") + parser.add_argument("--data_path", type = str, default = "", help = "") + parser.add_argument("--save_path", type = str, default = "", help = "") + parser.add_argument("--device_map", type = str, default = "1", help = "gpu or cpu") + return parser.parse_args() + +args = parse_args() +print(args) + +data = EmbDataset(args.data_path) +data_loader = DataLoader(data, num_workers = 4, batch_size = 64, shuffle = False, pin_memory = True) +device_map = {'': int(args.device_map)} +MODEL = LlamaWithRQ.from_pretrained(args.ckpt_path, torch_dtype = torch.float16, low_cpu_mem_usage = True, device_map = device_map) +MODEL.eval() +device = MODEL.device +rqvae = MODEL.rqvae +prefix = MODEL.prefix + +all_indices = [] +all_indices_str = [] +with torch.no_grad(): + for x in tqdm(data_loader): + indices = rqvae.get_indices(x.to(device), False) + indices = indices.view(-1, indices.shape[-1]).cpu().numpy() + for index in indices: + code = [] + for i, ind in enumerate(index): + code.append(prefix[i].format(int(ind))) + + all_indices.append(code) + all_indices_str.append(str(code)) + +all_indices = np.array(all_indices) +all_indices_str = np.array(all_indices_str) + +for vq in rqvae.rq.vq_layers[:-1]: + vq.sk_epsilon=0.0 +if rqvae.rq.vq_layers[-1].sk_epsilon == 0.0: + rqvae.rq.vq_layers[-1].sk_epsilon = 0.003 + +tt = 0 +while True: + if tt >= 20 or if_collided(all_indices_str): + break + + collision_item_groups = get_collision_item(all_indices_str) + # print(collision_item_groups) + print(len(collision_item_groups)) + with torch.no_grad(): + for collision_items in collision_item_groups: + indices = rqvae.get_indices(data[collision_items].to(device), True) + indices = indices.view(-1, indices.shape[-1]).cpu().numpy() + for item, index in zip(collision_items, indices): + code = [] + for i, ind in enumerate(index): + code.append(prefix[i].format(int(ind))) + + all_indices[item] = code + all_indices_str[item] = str(code) + tt += 1 + +print("All indices number: ",len(all_indices)) +print("Max number of conflicts: ", max(get_indices_count(all_indices_str).values())) + +tot_item = len(all_indices_str) +tot_indice = len(set(all_indices_str.tolist())) +print("Collision Rate",(tot_item - tot_indice) / tot_item) + +all_indices_dict = {} +for item, indices in enumerate(all_indices.tolist()): + all_indices_dict[item] = list(indices) + +with open(args.save_path, 'w',encoding = 'utf-8') as f: + json.dump(all_indices_dict, f) \ No newline at end of file diff --git a/index/datasets.py b/index/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..14c3ddec98d25b9c0f9d8a91eb5fa376d25a6be5 --- /dev/null +++ b/index/datasets.py @@ -0,0 +1,27 @@ +import numpy as np +import torch +import torch.utils.data as data +import pandas as pd +from tqdm import tqdm + +class EmbDataset(data.Dataset): + def __init__(self,data_path): + self.data_path = data_path + names = ['emb'] + usecols = [1] + tsv_data = pd.read_csv(data_path, sep = '\t',usecols = usecols, names = names, quotechar = None, quoting = 3) + features = tsv_data['emb'].values.tolist() + num_data = len(features) + for i in tqdm(range(num_data)): + features[i] = [float(s) for s in features[i].split(' ')] + self.embeddings = np.array(features, dtype = np.float16) + assert self.embeddings.shape[0] == num_data + self.dim = self.embeddings.shape[-1] + + def __getitem__(self, index): + emb = self.embeddings[index] + tensor_emb = torch.tensor(emb, dtype = torch.float16) + return tensor_emb + + def __len__(self): + return len(self.embeddings) diff --git a/index/generate_indices.py b/index/generate_indices.py new file mode 100644 index 0000000000000000000000000000000000000000..bd2220e8d7c58b933c1f4e1ceaa448f185fb43a4 --- /dev/null +++ b/index/generate_indices.py @@ -0,0 +1,151 @@ +import collections +import json +import logging +import argparse + +import numpy as np +import torch +from time import time +from torch import optim +from tqdm import tqdm + +from torch.utils.data import DataLoader + +from datasets import EmbDataset +from models.rqvae import RQVAE + +import os + +def check_collision(all_indices_str): + tot_item = len(all_indices_str) + tot_indice = len(set(all_indices_str.tolist())) + return tot_item==tot_indice + +def get_indices_count(all_indices_str): + indices_count = collections.defaultdict(int) + for index in all_indices_str: + indices_count[index] += 1 + return indices_count + +def get_collision_item(all_indices_str): + index2id = {} + for i, index in enumerate(all_indices_str): + if index not in index2id: + index2id[index] = [] + index2id[index].append(i) + + collision_item_groups = [] + + for index in index2id: + if len(index2id[index]) > 1: + collision_item_groups.append(index2id[index]) + + return collision_item_groups + +def parse_args(): + parser = argparse.ArgumentParser(description = "Index") + + parser.add_argument("--data_path", type = str, default = "", help = "Infer data path.") + parser.add_argument("--ckpt_path", type=str, default="", help="model checkpoint for infer") + parser.add_argument("--id_save_path", type=str, default="", help="output directory for id result") + parser.add_argument("--device", type=str, default="cuda:0", help="gpu or cpu") + + return parser.parse_args() + +# dataset = "Games" +# ckpt_path = "/zhengbowen/rqvae_ckpt/xxxx" +# output_dir = f"/zhengbowen/data/{dataset}/" +# output_file = f"{dataset}.index.json" +# output_file = os.path.join(output_dir,output_file) + +infer_args = parse_args() +print('infer_args:', infer_args) +device = torch.device(infer_args.device) +output_file = infer_args.id_save_path +data = EmbDataset(infer_args.data_path) + +ckpt = torch.load(infer_args.ckpt_path, map_location = torch.device('cpu')) +args = ckpt["args"] +state_dict = ckpt["state_dict"] + +model = RQVAE(in_dim=data.dim, + num_emb_list=args.num_emb_list, + e_dim=args.e_dim, + layers=args.layers, + dropout_prob=args.dropout_prob, + bn=args.bn, + loss_type=args.loss_type, + quant_loss_weight=args.quant_loss_weight, + kmeans_init=args.kmeans_init, + kmeans_iters=args.kmeans_iters, + sk_epsilons=args.sk_epsilons, + sk_iters=args.sk_iters, + ) + +model.load_state_dict(state_dict) +model = model.to(device) +model.eval() +print(model) + +data_loader = DataLoader(data, num_workers = args.num_workers, batch_size = 64, shuffle = False, pin_memory = True) + +all_indices = [] +all_indices_str = [] +prefix = ["","","","",""] + +for d in tqdm(data_loader): + d = d.to(device) + indices = model.get_indices(d,use_sk = False) + indices = indices.view(-1, indices.shape[-1]).cpu().numpy() + for index in indices: + code = [] + for i, ind in enumerate(index): + code.append(prefix[i].format(int(ind))) + + all_indices.append(code) + all_indices_str.append(str(code)) + +all_indices = np.array(all_indices) +all_indices_str = np.array(all_indices_str) + +for vq in model.rq.vq_layers[:-1]: + vq.sk_epsilon = 0.0 +if model.rq.vq_layers[-1].sk_epsilon == 0.0: + model.rq.vq_layers[-1].sk_epsilon = 0.003 + +tt = 0 +#There are often duplicate items in the dataset, and we no longer differentiate them +while True: + if tt >= 20 or check_collision(all_indices_str): + break + + collision_item_groups = get_collision_item(all_indices_str) + # print(collision_item_groups) + print(len(collision_item_groups)) + for collision_items in collision_item_groups: + d = data[collision_items].to(device) + + indices = model.get_indices(d, use_sk= True) + indices = indices.view(-1, indices.shape[-1]).cpu().numpy() + for item, index in zip(collision_items, indices): + code = [] + for i, ind in enumerate(index): + code.append(prefix[i].format(int(ind))) + + all_indices[item] = code + all_indices_str[item] = str(code) + tt += 1 + +print("All indices number: ", len(all_indices)) +print("Max number of conflicts: ", max(get_indices_count(all_indices_str).values())) + +tot_item = len(all_indices_str) +tot_indice = len(set(all_indices_str.tolist())) +print("Collision Rate", (tot_item - tot_indice) / tot_item) + +all_indices_dict = {} +for item, indices in enumerate(all_indices.tolist()): + all_indices_dict[item] = list(indices) + +with open(output_file, 'w') as fp: + json.dump(all_indices_dict, fp) diff --git a/index/main.py b/index/main.py new file mode 100644 index 0000000000000000000000000000000000000000..d6718ff95140ea304fd3302c34524ecb6fb6e8a5 --- /dev/null +++ b/index/main.py @@ -0,0 +1,87 @@ +import argparse +import random +import torch +import numpy as np +from time import time +import logging + +from torch.utils.data import DataLoader + +from datasets import EmbDataset +from models.rqvae import RQVAE +from trainer import Trainer + +def parse_args(): + parser = argparse.ArgumentParser(description="Index") + + parser.add_argument('--lr', type=float, default=1e-3, help='learning rate') + parser.add_argument('--epochs', type=int, default=5000, help='number of epochs') + parser.add_argument('--batch_size', type=int, default=1024, help='batch size') + parser.add_argument('--num_workers', type=int, default=4, ) + parser.add_argument('--eval_step', type=int, default=50, help='eval step') + parser.add_argument('--learner', type=str, default="AdamW", help='optimizer') + parser.add_argument("--data_path", type=str, + default="../data/Games/Games.emb-llama-td.npy", + help="Input data path.") + + parser.add_argument('--weight_decay', type=float, default=1e-4, help='l2 regularization weight') + parser.add_argument("--dropout_prob", type=float, default=0.0, help="dropout ratio") + parser.add_argument("--bn", type=bool, default=False, help="use bn or not") + parser.add_argument("--loss_type", type=str, default="mse", help="loss_type") + parser.add_argument("--kmeans_init", type=bool, default=True, help="use kmeans_init or not") + parser.add_argument("--kmeans_iters", type=int, default=100, help="max kmeans iters") + parser.add_argument('--sk_epsilons', type=float, nargs='+', default=[0.0, 0.0, 0.0], help="sinkhorn epsilons") + parser.add_argument("--sk_iters", type=int, default=50, help="max sinkhorn iters") + + parser.add_argument("--device", type=str, default="cuda:1", help="gpu or cpu") + + parser.add_argument('--num_emb_list', type=int, nargs='+', default=[256,256,256], help='emb num of every vq') + parser.add_argument('--e_dim', type=int, default=32, help='vq codebook embedding size') + parser.add_argument('--quant_loss_weight', type=float, default=1.0, help='vq quantion loss weight') + parser.add_argument('--layers', type=int, nargs='+', default=[2048,1024,512,256,128,64], help='hidden sizes of every layer') + + parser.add_argument("--ckpt_dir", type=str, default="", help="output directory for model") + + return parser.parse_args() + + +if __name__ == '__main__': + """fix the random seed""" + seed = 2023 + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + args = parse_args() + print(args) + + logging.basicConfig(level=logging.DEBUG) + + """build dataset""" + data = EmbDataset(args.data_path) + model = RQVAE(in_dim=data.dim, + num_emb_list=args.num_emb_list, + e_dim=args.e_dim, + layers=args.layers, + dropout_prob=args.dropout_prob, + bn=args.bn, + loss_type=args.loss_type, + quant_loss_weight=args.quant_loss_weight, + kmeans_init=args.kmeans_init, + kmeans_iters=args.kmeans_iters, + sk_epsilons=args.sk_epsilons, + sk_iters=args.sk_iters, + ) + print(model) + data_loader = DataLoader(data,num_workers=args.num_workers, + batch_size=args.batch_size, shuffle=True, + pin_memory=True) + trainer = Trainer(args,model) + best_loss, best_collision_rate = trainer.fit(data_loader) + + print("Best Loss",best_loss) + print("Best Collision Rate", best_collision_rate) + diff --git a/index/run.sh b/index/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..7462b6ca9622206de7085b67f9221d425332984d --- /dev/null +++ b/index/run.sh @@ -0,0 +1,8 @@ + +python -u main.py \ + --num_emb_list 256 256 256 256 \ + --sk_epsilons 0.0 0.0 0.0 0.003 \ + --device cuda:0 \ + --data_path /data/Games/Games.emb-llama-td.npy \ + --batch_size 1024 + diff --git a/index/trainer.py b/index/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..eb4d5b7f8fbb6637a1f6932101402b72ec923db9 --- /dev/null +++ b/index/trainer.py @@ -0,0 +1,209 @@ +import logging + +import numpy as np +import torch +from time import time +from torch import optim +from tqdm import tqdm + +from utils import ensure_dir,set_color,get_local_time +import os + +class Trainer(object): + + def __init__(self, args, model): + self.args = args + self.model = model + self.logger = logging.getLogger() + + self.lr = args.lr + self.learner = args.learner + self.weight_decay = args.weight_decay + self.epochs = args.epochs + self.eval_step = min(args.eval_step, self.epochs) + self.device = args.device + self.device = torch.device(self.device) + self.ckpt_dir = args.ckpt_dir + saved_model_dir = "{}".format(get_local_time()) + self.ckpt_dir = os.path.join(self.ckpt_dir,saved_model_dir) + ensure_dir(self.ckpt_dir) + + self.best_loss = np.inf + self.best_collision_rate = np.inf + self.best_loss_ckpt = "best_loss_model.pth" + self.best_collision_ckpt = "best_collision_model.pth" + self.optimizer = self._build_optimizer() + self.model = self.model.to(self.device) + + def _build_optimizer(self): + + params = self.model.parameters() + learner = self.learner + learning_rate = self.lr + weight_decay = self.weight_decay + + if learner.lower() == "adam": + optimizer = optim.Adam(params, lr=learning_rate, weight_decay=weight_decay) + elif learner.lower() == "sgd": + optimizer = optim.SGD(params, lr=learning_rate, weight_decay=weight_decay) + elif learner.lower() == "adagrad": + optimizer = optim.Adagrad( + params, lr=learning_rate, weight_decay=weight_decay + ) + for state in optimizer.state.values(): + for k, v in state.items(): + if torch.is_tensor(v): + state[k] = v.to(self.device) + elif learner.lower() == "rmsprop": + optimizer = optim.RMSprop( + params, lr=learning_rate, weight_decay=weight_decay + ) + elif learner.lower() == 'adamw': + optimizer = optim.AdamW( + params, lr=learning_rate, weight_decay=weight_decay + ) + else: + self.logger.warning( + "Received unrecognized optimizer, set default Adam optimizer" + ) + optimizer = optim.Adam(params, lr=learning_rate) + return optimizer + def _check_nan(self, loss): + if torch.isnan(loss): + raise ValueError("Training loss is nan") + + def _train_epoch(self, train_data, epoch_idx): + + self.model.train() + + total_loss = 0 + total_recon_loss = 0 + iter_data = tqdm( + train_data, + total=len(train_data), + ncols=100, + desc=set_color(f"Train {epoch_idx}","pink"), + ) + + for batch_idx, data in enumerate(iter_data): + data = data.to(self.device) + self.optimizer.zero_grad() + out, rq_loss, indices = self.model(data) + loss, loss_recon = self.model.compute_loss(out, rq_loss, xs=data) + self._check_nan(loss) + loss.backward() + self.optimizer.step() + total_loss += loss.item() + total_recon_loss += loss_recon.item() + + return total_loss, total_recon_loss + + @torch.no_grad() + def _valid_epoch(self, valid_data): + + self.model.eval() + + iter_data =tqdm( + valid_data, + total=len(valid_data), + ncols=100, + desc=set_color(f"Evaluate ", "pink"), + ) + indices_set = set() + num_sample = 0 + for batch_idx, data in enumerate(iter_data): + num_sample += len(data) + data = data.to(self.device) + indices = self.model.get_indices(data) + indices = indices.view(-1,indices.shape[-1]).cpu().numpy() + for index in indices: + code = "-".join([str(int(_)) for _ in index]) + indices_set.add(code) + + collision_rate = (num_sample - len(indices_set))/num_sample + + return collision_rate + + def _save_checkpoint(self, epoch, collision_rate=1, ckpt_file=None): + + ckpt_path = os.path.join(self.ckpt_dir,ckpt_file) if ckpt_file \ + else os.path.join(self.ckpt_dir, 'epoch_%d_collision_%.4f_model.pth' % (epoch, collision_rate)) + state = { + "args": self.args, + "epoch": epoch, + "best_loss": self.best_loss, + "best_collision_rate": self.best_collision_rate, + "state_dict": self.model.state_dict(), + "optimizer": self.optimizer.state_dict(), + } + torch.save(state, ckpt_path, pickle_protocol=4) + + self.logger.info( + set_color("Saving current", "blue") + f": {ckpt_path}" + ) + + def _generate_train_loss_output(self, epoch_idx, s_time, e_time, loss, recon_loss): + train_loss_output = ( + set_color("epoch %d training", "green") + + " [" + + set_color("time", "blue") + + ": %.2fs, " + ) % (epoch_idx, e_time - s_time) + train_loss_output += set_color("train loss", "blue") + ": %.4f" % loss + train_loss_output +=", " + train_loss_output += set_color("reconstruction loss", "blue") + ": %.4f" % recon_loss + return train_loss_output + "]" + + + def fit(self, data): + + cur_eval_step = 0 + + for epoch_idx in range(self.epochs): + # train + training_start_time = time() + train_loss, train_recon_loss = self._train_epoch(data, epoch_idx) + training_end_time = time() + train_loss_output = self._generate_train_loss_output( + epoch_idx, training_start_time, training_end_time, train_loss, train_recon_loss + ) + self.logger.info(train_loss_output) + + if train_loss < self.best_loss: + self.best_loss = train_loss + # self._save_checkpoint(epoch=epoch_idx,ckpt_file=self.best_loss_ckpt) + + # eval + if (epoch_idx + 1) % self.eval_step == 0: + valid_start_time = time() + collision_rate = self._valid_epoch(data) + + if collision_rate < self.best_collision_rate: + self.best_collision_rate = collision_rate + cur_eval_step = 0 + self._save_checkpoint(epoch_idx, collision_rate=collision_rate, + ckpt_file=self.best_collision_ckpt) + else: + cur_eval_step += 1 + + + valid_end_time = time() + valid_score_output = ( + set_color("epoch %d evaluating", "green") + + " [" + + set_color("time", "blue") + + ": %.2fs, " + + set_color("collision_rate", "blue") + + ": %f]" + ) % (epoch_idx, valid_end_time - valid_start_time, collision_rate) + + self.logger.info(valid_score_output) + if epoch_idx>1000: + self._save_checkpoint(epoch_idx, collision_rate=collision_rate) + + + return self.best_loss, self.best_collision_rate + + + + diff --git a/index/utils.py b/index/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..15464c7a6d1a975e26fdc6b0430d24edf0722596 --- /dev/null +++ b/index/utils.py @@ -0,0 +1,36 @@ + +import datetime +import os + + +def ensure_dir(dir_path): + + os.makedirs(dir_path, exist_ok=True) + +def set_color(log, color, highlight=True): + color_set = ["black", "red", "green", "yellow", "blue", "pink", "cyan", "white"] + try: + index = color_set.index(color) + except: + index = len(color_set) - 1 + prev_log = "\033[" + if highlight: + prev_log += "1;3" + else: + prev_log += "0;3" + prev_log += str(index) + "m" + return prev_log + log + "\033[0m" + +def get_local_time(): + r"""Get current time + + Returns: + str: current time + """ + cur = datetime.datetime.now() + cur = cur.strftime("%b-%d-%Y_%H-%M-%S") + + return cur + + + diff --git a/infer.sh b/infer.sh new file mode 100644 index 0000000000000000000000000000000000000000..4b5461ac400b4afd54eaff6f01b8bd9b9a60121d --- /dev/null +++ b/infer.sh @@ -0,0 +1,12 @@ +CKPT_PATH=/home/jovyan/workspace/ed2/Ins + +python generate_embeddings.py \ + --ckpt_path $CKPT_PATH \ + --save_path $CKPT_PATH/embeddings.tsv \ + --device_map 0 + +python generate_indices_v2.py \ + --ckpt_path $CKPT_PATH \ + --data_path $CKPT_PATH/embeddings.tsv \ + --save_path $CKPT_PATH \ + --device_map 0 diff --git a/instruments_evaluate.sh b/instruments_evaluate.sh new file mode 100644 index 0000000000000000000000000000000000000000..66bc8efcd5c7b15037f0d75b1c8b139a9dd1710c --- /dev/null +++ b/instruments_evaluate.sh @@ -0,0 +1,18 @@ +DATASET=Instruments +BASE_MODEL=/home/jovyan/workspace/Llama-7b +DATA_PATH=/home/jovyan/workspace +CKPT_PATH=/home/jovyan/workspace/ed2/Ins/finetune +RESULTS_FILE=$CKPT_PATH/eval_result.json +INDEX=/home/jovyan/workspace/ed2/Ins/indices.json + +torchrun --nproc_per_node=4 evaluate-finetuned.py \ + --base_model $BASE_MODEL \ + --ckpt_path $CKPT_PATH \ + --dataset $DATASET \ + --data_path $DATA_PATH \ + --results_file $RESULTS_FILE \ + --test_batch_size 1 \ + --num_beams 20 \ + --test_prompt_ids all \ + --test_task seqrec \ + --index_file $INDEX diff --git a/instruments_finetune.sh b/instruments_finetune.sh new file mode 100644 index 0000000000000000000000000000000000000000..d5c5bd575f21186cb7865ab7902236892c5c0359 --- /dev/null +++ b/instruments_finetune.sh @@ -0,0 +1,27 @@ +export WANDB_MODE=disabled +export CUDA_LAUNCH_BLOCKING=0 + +DATASET=Instruments +CKPT_PATH=/home/jovyan/workspace/ed2/Ins +DATA_PATH=/home/jovyan/workspace +OUTPUT_DIR=$CKPT_PATH/finetune + +torchrun --nproc_per_node=4 fine-tune.py \ + --ckpt_path $CKPT_PATH \ + --output_dir $OUTPUT_DIR \ + --dataset $DATASET \ + --data_path $DATA_PATH \ + --per_device_batch_size 8 \ + --gradient_accumulation_steps 2 \ + --learning_rate 5e-4 \ + --epochs 1 \ + --weight_decay 0.01 \ + --save_and_eval_strategy epoch \ + --bf16 \ + --deepspeed ./config/ds_z2_bf16.json \ + --dataloader_num_workers 4 \ + --only_train_response \ + --tasks seqrec,itemsearch,preferenceobtain,item2index,index2item,fusionseqrec \ + --train_prompt_sample_num 1,1,1,1,1,1 \ + --train_data_sample_num 0,0,0,0,0,0 \ + --index_file $CKPT_PATH/indices.json &>>$OUTPUT_DIR/log.txt diff --git a/instruments_more_pretrain.sh b/instruments_more_pretrain.sh new file mode 100644 index 0000000000000000000000000000000000000000..129ebdd5161d2f278c14ce922f6daaf409cb5485 --- /dev/null +++ b/instruments_more_pretrain.sh @@ -0,0 +1,32 @@ +export WANDB_MODE=disabled +export CUDA_LAUNCH_BLOCKING=0 + +DATASET=Instruments +BASE_MODEL=/datain/v-yinju/llama-7b +CKPT_PATH=/datain/v-yinju/rq-llama/v3-train/Instruments/first +DATA_PATH=/datain/v-yinju/rqvae-zzx/data +OUTPUT_DIR=/datain/v-yinju/rq-llama/v3-train/Instruments/more_pretrain + +torchrun --nproc_per_node=8 --master_port=3324 continue_pretrain.py \ + --base_model $BASE_MODEL \ + --rqvae_model $CKPT_PATH \ + --output_dir $OUTPUT_DIR \ + --dataset $DATASET \ + --data_path $DATA_PATH \ + --per_device_batch_size 6 \ + --gradient_accumulation_steps 2 \ + --learning_rate 5e-5 \ + --epochs 4 \ + --weight_decay 0.01 \ + --save_and_eval_strategy epoch \ + --deepspeed ./config/ds_z2_fp16.json \ + --dataloader_num_workers 4 \ + --only_train_response \ + --tasks seqrec,itemsearch,inters2title,inters2description,preferenceobtain,item2index,index2item,intertitles2item,query2item \ + --train_prompt_sample_num 1,1,1,1,1,1,1,1,1 \ + --train_data_sample_num 0,0,0,0,0,0,0,0,0 \ + --fp16 &>>$OUTPUT_DIR/pretrain-log.txt + +cd convert +nohup ./convert.sh $OUTPUT_DIR >convert.log 2>&1 & +cd .. \ No newline at end of file diff --git a/lora_finetune.py b/lora_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..612faaeeb1ec9780c446b819a9b94064a4a60214 --- /dev/null +++ b/lora_finetune.py @@ -0,0 +1,162 @@ +import argparse +import os +import sys +from typing import List + +import torch +import transformers + + +from peft import ( + TaskType, + LoraConfig, + get_peft_model, + get_peft_model_state_dict, + set_peft_model_state_dict, +) +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig + +from utils import * +from collator import Collator + +def train(args): + + set_seed(args.seed) + ensure_dir(args.output_dir) + + device_map = "auto" + world_size = int(os.environ.get("WORLD_SIZE", 1)) + ddp = world_size != 1 + local_rank = int(os.environ.get("LOCAL_RANK") or 0) + if local_rank == 0: + print(vars(args)) + + if ddp: + device_map = {"": local_rank} + + config = LlamaConfig.from_pretrained(args.base_model) + tokenizer = LlamaTokenizer.from_pretrained( + args.base_model, + model_max_length=args.model_max_length, + padding_side="right", + ) + tokenizer.pad_token_id = 0 + + train_data, valid_data = load_datasets(args) + add_num = tokenizer.add_tokens(train_data.datasets[0].get_new_tokens()) + config.vocab_size = len(tokenizer) + if local_rank == 0: + print("add {} new token.".format(add_num)) + print("data num:", len(train_data)) + tokenizer.save_pretrained(args.output_dir) + config.save_pretrained(args.output_dir) + + collator = Collator(args, tokenizer) + + model = LlamaForCausalLM.from_pretrained( + args.base_model, + # torch_dtype=torch.float16, + device_map=device_map, + ) + model.resize_token_embeddings(len(tokenizer)) + + config = LoraConfig( + r=args.lora_r, + lora_alpha=args.lora_alpha, + target_modules=args.lora_target_modules.split(","), + modules_to_save=args.lora_modules_to_save.split(","), + lora_dropout=args.lora_dropout, + bias="none", + inference_mode=False, + task_type=TaskType.CAUSAL_LM, + ) + model = get_peft_model(model, config) + + if args.resume_from_checkpoint: + checkpoint_name = os.path.join( + args.resume_from_checkpoint, "adapter_model.bin" + ) # only LoRA model - LoRA config above has to fit + args.resume_from_checkpoint = False # So the trainer won't try loading its state + # The two files above have a different name depending on how they were saved, but are actually the same. + if os.path.exists(checkpoint_name): + if local_rank == 0: + print(f"Restarting from {checkpoint_name}") + adapters_weights = torch.load(checkpoint_name) + model = set_peft_model_state_dict(model, adapters_weights) + else: + if local_rank == 0: + print(f"Checkpoint {checkpoint_name} not found") + + for n, p in model.named_parameters(): + if "original_module" in n and any(module_name in n for module_name in config.modules_to_save): + p.requires_grad = False + + if local_rank == 0: + model.print_trainable_parameters() + + + if not ddp and torch.cuda.device_count() > 1: + model.is_parallelizable = True + model.model_parallel = True + + trainer = transformers.Trainer( + model=model, + train_dataset=train_data, + eval_dataset=valid_data, + args=transformers.TrainingArguments( + seed=args.seed, + per_device_train_batch_size=args.per_device_batch_size, + per_device_eval_batch_size=args.per_device_batch_size, + gradient_accumulation_steps=args.gradient_accumulation_steps, + warmup_ratio=args.warmup_ratio, + num_train_epochs=args.epochs, + learning_rate=args.learning_rate, + weight_decay=args.weight_decay, + lr_scheduler_type=args.lr_scheduler_type, + fp16=args.fp16, + bf16=args.bf16, + logging_steps=args.logging_step, + optim=args.optim, + gradient_checkpointing=True, + evaluation_strategy=args.save_and_eval_strategy, + save_strategy=args.save_and_eval_strategy, + eval_steps=args.save_and_eval_steps, + save_steps=args.save_and_eval_steps, + output_dir=args.output_dir, + save_total_limit=5, + load_best_model_at_end=True, + deepspeed=args.deepspeed, + ddp_find_unused_parameters=False if ddp else None, + report_to=None, + eval_delay=1 if args.save_and_eval_strategy=="epoch" else 2000, + ), + tokenizer=tokenizer, + data_collator=collator, + ) + model.config.use_cache = False + + # old_state_dict = model.state_dict + # model.state_dict = ( + # lambda self, *_, **__: get_peft_model_state_dict(self, old_state_dict()) + # ).__get__(model, type(model)) + + if torch.__version__ >= "2" and sys.platform != "win32": + model = torch.compile(model) + + trainer.train( + resume_from_checkpoint=args.resume_from_checkpoint, + ) + + trainer.save_state() + trainer.save_model(output_dir=args.output_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='LLMRec') + parser = parse_global_args(parser) + parser = parse_train_args(parser) + parser = parse_dataset_args(parser) + + args = parser.parse_args() + + train(args) diff --git a/prompt.py b/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..87c9409d06d2e0a382820a709aa97ba4dfabf41f --- /dev/null +++ b/prompt.py @@ -0,0 +1,664 @@ + + +sft_prompt = "Below is an instruction that describes a task. Write a response that appropriately completes the request." \ + "\n\n### Instruction:\n{instruction} Please think it step by step.\n\n### Response:{response}" + + + + + + + +all_prompt = {} + +# ===================================================== +# Task 1 -- Sequential Recommendation -- 17 Prompt +# ===================================================== + +seqrec_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "The user has interacted with items {inters} in chronological order. Can you predict the next possible item that the user may expect?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "I find the user's historical interactive items: {inters}, and I want to know what next item the user needs. Can you help me decide?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "Here are the user's historical interactions: {inters}, try to recommend another item to the user. Note that the historical interactions are arranged in chronological order." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Based on the items that the user has interacted with: {inters}, can you determine what item would be recommended to him next?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "The user has interacted with the following items in order: {inters}. What else do you think the user need?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Here is the item interaction history of the user: {inters}, what to recommend to the user next?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——6 +prompt = {} +prompt["instruction"] = "Which item would the user be likely to interact with next after interacting with items {inters}?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "By analyzing the user's historical interactions with items {inters}, what is the next expected interaction item?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "After interacting with items {inters}, what is the next item that could be recommended for the user?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "Given the user's historical interactive items arranged in chronological order: {inters}, can you recommend a suitable item for the user?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "Considering the user has interacted with items {inters}. What is the next recommendation for the user?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "What is the top recommended item for the user who has previously interacted with items {inters} in order?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——12 +prompt = {} +prompt["instruction"] = "The user has interacted with the following items in the past in order: {inters}. Please predict the next item that the user most desires based on the given interaction records." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +# prompt = {} +# prompt["instruction"] = "The user has interacted with the following items in the past in order: {inters}. Please predict the next item that the user is most likely to interact with based on the given interaction record. Note that his most recently interacted item is {}." +# prompt["response"] = "{item}" +# prompt["task"] = "sequential" +# prompt["id"] = "1-13" +# +# seqrec_prompt.append(prompt) + +#####——13 +prompt = {} +prompt["instruction"] = "Using the user's historical interactions as input data, suggest the next item that the user is highly likely to enjoy. The historical interactions are provided as follows: {inters}." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——14 +prompt = {} +prompt["instruction"] = "You can access the user's historical item interaction records: {inters}. Now your task is to recommend the next potential item to him, considering his past interactions." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——15 +prompt = {} +prompt["instruction"] = "You have observed that the user has interacted with the following items: {inters}, please recommend a next item that you think would be suitable for the user." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——16 +prompt = {} +prompt["instruction"] = "You have obtained the ordered list of user historical interaction items, which is as follows: {inters}. Using this history as a reference, please select the next item to recommend to the user." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +all_prompt["seqrec"] = seqrec_prompt + + + +# ======================================================== +# Task 2 -- Item2Index -- 19 Prompt +# ======================================================== +# Remove periods when inputting + +item2index_prompt = [] + +# ======================================================== +# Title2Index + +#####——0 +prompt = {} +prompt["instruction"] = "Which item has the title: \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "Which item is assigned the title: \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "An item is called \"{title}\", could you please let me know which item it is?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Which item is called \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "One of the items is named \"{title}\", can you tell me which item this is?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "What is the item that goes by the title \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +# prompt = {} +# prompt["instruction"] = "Which item is referred to as \"{title}\"?" +# prompt["response"] = "{item}" +# item2index_prompt.append(prompt) + +# ======================================================== +# Description2Index + +#####——6 +prompt = {} +prompt["instruction"] = "An item can be described as follows: \"{description}\". Which item is it describing?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "Can you tell me what item is described as \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "Can you provide the item that corresponds to the following description: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + + +# prompt = {} +# prompt["instruction"] = "What is the item described as follows: \"{description}\"?" +# prompt["response"] = "{item}" +# item2index_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "Which item has the following characteristics: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "Which item is characterized by the following description: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "I am curious to know which item can be described as follows: \"{description}\". Can you tell me?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +# ======================================================== +# Title and Description to index + +#####——12 +prompt = {} +prompt["instruction"] = "An item is called \"{title}\" and described as \"{description}\", can you tell me which item it is?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——13 +prompt = {} +prompt["instruction"] = "Could you please identify what item is called \"{title}\" and described as \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——14 +prompt = {} +prompt["instruction"] = "Which item is called \"{title}\" and has the characteristics described below: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——15 +prompt = {} +prompt["instruction"] = "Please show me which item is named \"{title}\" and its corresponding description is: \"{description}\"." +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + + +# prompt = {} +# prompt["instruction"] = "Here is an item called \"{title}\" and described as \"{description}\". Which item is it?" +# prompt["response"] = "{item}" +# item2index_prompt.append(prompt) + +#####——16 +prompt = {} +prompt["instruction"] = "Determine which item this is by its title and description. The title is: \"{title}\", and the description is: \"{description}\"." +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——17 +prompt = {} +prompt["instruction"] = "Based on the title: \"{title}\", and the description: \"{description}\", answer which item is this?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——18 +prompt = {} +prompt["instruction"] = "Can you identify the item from the provided title: \"{title}\", and description: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +all_prompt["item2index"] = item2index_prompt + + +# ======================================================== +# Task 3 -- Index2Item --17 Prompt +# ======================================================== +# Remove periods when inputting + +index2item_prompt = [] + +# ======================================================== +# Index2Title + +#####——0 +prompt = {} +prompt["instruction"] = "What is the title of item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "What title is assigned to item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "Could you please tell me what item {item} is called?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Can you provide the title of item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "What item {item} is referred to as?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Would you mind informing me about the title of item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +# ======================================================== +# Index2Description + +#####——6 +prompt = {} +prompt["instruction"] = "Please provide a description of item {item}." +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "Briefly describe item {item}." +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "Can you share with me the description corresponding to item {item}?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "What is the description of item {item}?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "How to describe the characteristics of item {item}?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "Could you please tell me what item {item} looks like?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + + +# ======================================================== +# index to Title and Description + +#####——12 +prompt = {} +prompt["instruction"] = "What is the title and description of item {item}?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——13 +prompt = {} +prompt["instruction"] = "Can you provide the corresponding title and description for item {item}?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——14 +prompt = {} +prompt["instruction"] = "Please tell me what item {item} is called, along with a brief description of it." +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——15 +prompt = {} +prompt["instruction"] = "Would you mind informing me about the title of the item {item} and how to describe its characteristics?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——16 +prompt = {} +prompt["instruction"] = "I need to know the title and description of item {item}. Could you help me with that?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +all_prompt["index2item"] = index2item_prompt + + + + + +# ======================================================== +# Task 4 -- Interactions2Title -- Prompt +# ======================================================== + + +inters2title_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "The user has sequentially interacted with items {inters}. Can you recommend the next item for him? Tell me the title of the item？" +prompt["response"] = "{title}" +inters2title_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "Based on the user's historical interactions: {inters}, try to predict the title of the item that the user may need next." +prompt["response"] = "{title}" +inters2title_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "Utilizing the user's past ordered interactions, which include items {inters}, please recommend the next item you think is suitable for the user and provide its title." +prompt["response"] = "{title}" +inters2title_prompt.append(prompt) + + +#####——3 +prompt = {} +prompt["instruction"] = "After interacting with items {inters}, what is the most probable item for the user to interact with next? Kindly provide the item's title." +prompt["response"] = "{title}" +inters2title_prompt.append(prompt) + +all_prompt["inters2title"] = inters2title_prompt + + +# ======================================================== +# Task 5 -- Interactions2Description -- Prompt +# ======================================================== + +inters2description_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "Please review the user's historical interactions: {inters}, and describe what kind of item he still needs." +prompt["response"] = "{description}" +inters2description_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "Here is the item interaction history of the user: {inters}, please tell me what features he expects from his next item." +prompt["response"] = "{description}" +inters2description_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "By analyzing the user's historical interactions with items {inters}, can you infer what the user's next interactive item will look like?" +prompt["response"] = "{description}" +inters2description_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Access the user's historical item interaction records: {inters}. Your objective is to describe the next potential item for him, taking into account his past interactions." +prompt["response"] = "{description}" +inters2description_prompt.append(prompt) + +all_prompt["inters2description"] = inters2description_prompt + + +# ======================================================== +# Task 6 -- InteractedTitles2Item -- Prompt +# ======================================================== + +intertitles2item_prompt = [] +#####——0 +prompt = {} +prompt["instruction"] = "Given the title sequence of user historical interactive items: {inter_titles}, can you recommend a suitable next item for the user?" +prompt["response"] = "{item}" +intertitles2item_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "I possess a user's past interaction history, denoted by the title sequence of interactive items: {inter_titles}, and I am interested in knowing the user's next most desired item. Can you help me?" +prompt["response"] = "{item}" +intertitles2item_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "Considering the title sequence of user history interaction items: {inter_titles}. What is the next recommendation for the user?" +prompt["response"] = "{item}" +intertitles2item_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "You have obtained the ordered title list of user historical interaction items, as follows: {inter_titles}. Based on this historical context, kindly choose the subsequent item for user recommendation." +prompt["response"] = "{item}" +intertitles2item_prompt.append(prompt) + +all_prompt["intertitles2item"] = intertitles2item_prompt + + +# ======================================================== +# Task 7 -- ItemSearch -- Prompt +# ======================================================== + +itemsearch_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "Here is the historical interactions of a user: {inters}. And his personalized preferences are as follows: \"{explicit_preference}\". Your task is to recommend an item that is consistent with the user's preference." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "The user has interacted with a list of items, which are as follows: {inters}. Based on these interacted items, the user current intent is as follows \"{user_related_intention}\", and your task is to generate an item that matches the user's current intent." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "As a recommender system, you are assisting a user who has recently interacted with the following items: {inters}. The user expresses a desire to obtain another item with the following characteristics: \"{item_related_intention}\". Please recommend an item that meets these criteria." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Using the user's current query: \"{query}\" and his historical interactions: {inters}, you can estimate the user's preferences \"{explicit_preference}\". Please respond to the user's query by selecting an item that best matches his preference and query." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "The user needs a new item and searches for: \"{query}\". In addition, he has previously interacted with: {inters}. You can obtain his preference by analyzing his historical interactions: \"{explicit_preference}\". Can you recommend an item that best matches the search query and preferences?" +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Based on the user's historical interactions with the following items: {inters}. You can infer his preference by observing the historical interactions: \"{explicit_preference}\". Now the user wants a new item and searches for: \"{query}\". Please select a suitable item that matches his preference and search intent." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +all_prompt["itemsearch"] = itemsearch_prompt + +# ======================================================== +# Task 8 -- Query2Item -- Prompt +# ======================================================== + +query2item_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "Suppose you are a search engine, now a user searches that: \"{query}\", can you select an item to respond to the user's query?" +prompt["response"] = "{item}" +query2item_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "As a search engine, your task is to answer the user's query by generating a related item. The user's query is provided as \"{query}\". Please provide your generated item as your answer." +prompt["response"] = "{item}" +query2item_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "As a recommender system, your task is to recommend an item that is related to the user's request, which is specified as follows: \"{query}\". Please provide your recommendation." +prompt["response"] = "{item}" +query2item_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "You meet a user's query: \"{query}\". Please respond to this user by selecting an appropriate item." +prompt["response"] = "{item}" +query2item_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "Your task is to recommend the best item that matches the user's query. Here is the search query of the user: \"{query}\", tell me the item you recommend." +prompt["response"] = "{item}" +query2item_prompt.append(prompt) + +all_prompt["query2item"] = query2item_prompt + + +# ======================================================== +# Task 9 -- PreferenceObtain -- Prompt +# ======================================================== + +preferenceobtain_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "The user has interacted with items {inters} in chronological order. Please estimate his preferences." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "Based on the items that the user has interacted with: {inters}, can you infer what preferences he has?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Can you provide a summary of the user's preferences based on his historical interactions: {inters}?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "After interacting with items {inters} in order, what preferences do you think the user has?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Here is the item interaction history of the user: {inters}, could you please infer the user's preferences." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——6 +prompt = {} +prompt["instruction"] = "Based on the user's historical interaction records: {inters}, what are your speculations about his preferences?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "Given the user's historical interactive items arranged in chronological order: {inters}, what can be inferred about the preferences of the user?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "Can you speculate on the user's preferences based on his historical item interaction records: {inters}?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "What is the preferences of a user who has previously interacted with items {inters} sequentially?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "Using the user's historical interactions as input data, summarize the user's preferences. The historical interactions are provided as follows: {inters}." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "Utilizing the ordered list of the user's historical interaction items as a reference, please make an informed estimation of the user's preferences. The historical interactions are as follows: {inters}." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +all_prompt["preferenceobtain"] = preferenceobtain_prompt diff --git a/prompt_finetune.py b/prompt_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..0f3f8dd889dbef056c99c91e1ef249f293ab763a --- /dev/null +++ b/prompt_finetune.py @@ -0,0 +1,663 @@ + + +sft_prompt = "Below is an instruction that describes a task. Write a response that appropriately completes the request." \ + "\n\n### Instruction:\n{instruction}\n\n### Response:{response}" + + + + + + + +all_prompt = {} + +# ===================================================== +# Task 1 -- Sequential Recommendation -- 17 Prompt +# ===================================================== + +seqrec_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "The user has interacted with items {inters} in chronological order. Can you predict the next possible item that the user may expect?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "I find the user's historical interactive items: {inters}, and I want to know what next item the user needs. Can you help me decide?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "Here are the user's historical interactions: {inters}, try to recommend another item to the user. Note that the historical interactions are arranged in chronological order." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Based on the items that the user has interacted with: {inters}, can you determine what item would be recommended to him next?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "The user has interacted with the following items in order: {inters}. What else do you think the user need?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Here is the item interaction history of the user: {inters}, what to recommend to the user next?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——6 +prompt = {} +prompt["instruction"] = "Which item would the user be likely to interact with next after interacting with items {inters}?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "By analyzing the user's historical interactions with items {inters}, what is the next expected interaction item?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "After interacting with items {inters}, what is the next item that could be recommended for the user?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "Given the user's historical interactive items arranged in chronological order: {inters}, can you recommend a suitable item for the user?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "Considering the user has interacted with items {inters}. What is the next recommendation for the user?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "What is the top recommended item for the user who has previously interacted with items {inters} in order?" +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——12 +prompt = {} +prompt["instruction"] = "The user has interacted with the following items in the past in order: {inters}. Please predict the next item that the user most desires based on the given interaction records." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +# prompt = {} +# prompt["instruction"] = "The user has interacted with the following items in the past in order: {inters}. Please predict the next item that the user is most likely to interact with based on the given interaction record. Note that his most recently interacted item is {}." +# prompt["response"] = "{item}" +# prompt["task"] = "sequential" +# prompt["id"] = "1-13" +# +# seqrec_prompt.append(prompt) + +#####——13 +prompt = {} +prompt["instruction"] = "Using the user's historical interactions as input data, suggest the next item that the user is highly likely to enjoy. The historical interactions are provided as follows: {inters}." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——14 +prompt = {} +prompt["instruction"] = "You can access the user's historical item interaction records: {inters}. Now your task is to recommend the next potential item to him, considering his past interactions." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——15 +prompt = {} +prompt["instruction"] = "You have observed that the user has interacted with the following items: {inters}, please recommend a next item that you think would be suitable for the user." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +#####——16 +prompt = {} +prompt["instruction"] = "You have obtained the ordered list of user historical interaction items, which is as follows: {inters}. Using this history as a reference, please select the next item to recommend to the user." +prompt["response"] = "{item}" +seqrec_prompt.append(prompt) + +all_prompt["seqrec"] = seqrec_prompt + + + +# ======================================================== +# Task 2 -- Item2Index -- 19 Prompt +# ======================================================== +# Remove periods when inputting + +item2index_prompt = [] + +# ======================================================== +# Title2Index + +#####——0 +prompt = {} +prompt["instruction"] = "Which item has the title: \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "Which item is assigned the title: \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "An item is called \"{title}\", could you please let me know which item it is?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Which item is called \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "One of the items is named \"{title}\", can you tell me which item this is?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "What is the item that goes by the title \"{title}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +# prompt = {} +# prompt["instruction"] = "Which item is referred to as \"{title}\"?" +# prompt["response"] = "{item}" +# item2index_prompt.append(prompt) + +# ======================================================== +# Description2Index + +#####——6 +prompt = {} +prompt["instruction"] = "An item can be described as follows: \"{description}\". Which item is it describing?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "Can you tell me what item is described as \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "Can you provide the item that corresponds to the following description: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + + +# prompt = {} +# prompt["instruction"] = "What is the item described as follows: \"{description}\"?" +# prompt["response"] = "{item}" +# item2index_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "Which item has the following characteristics: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "Which item is characterized by the following description: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "I am curious to know which item can be described as follows: \"{description}\". Can you tell me?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +# ======================================================== +# Title and Description to index + +#####——12 +prompt = {} +prompt["instruction"] = "An item is called \"{title}\" and described as \"{description}\", can you tell me which item it is?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——13 +prompt = {} +prompt["instruction"] = "Could you please identify what item is called \"{title}\" and described as \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——14 +prompt = {} +prompt["instruction"] = "Which item is called \"{title}\" and has the characteristics described below: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——15 +prompt = {} +prompt["instruction"] = "Please show me which item is named \"{title}\" and its corresponding description is: \"{description}\"." +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + + +# prompt = {} +# prompt["instruction"] = "Here is an item called \"{title}\" and described as \"{description}\". Which item is it?" +# prompt["response"] = "{item}" +# item2index_prompt.append(prompt) + +#####——16 +prompt = {} +prompt["instruction"] = "Determine which item this is by its title and description. The title is: \"{title}\", and the description is: \"{description}\"." +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——17 +prompt = {} +prompt["instruction"] = "Based on the title: \"{title}\", and the description: \"{description}\", answer which item is this?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +#####——18 +prompt = {} +prompt["instruction"] = "Can you identify the item from the provided title: \"{title}\", and description: \"{description}\"?" +prompt["response"] = "{item}" +item2index_prompt.append(prompt) + +all_prompt["item2index"] = item2index_prompt + + +# ======================================================== +# Task 3 -- Index2Item --17 Prompt +# ======================================================== +# Remove periods when inputting + +index2item_prompt = [] + +# ======================================================== +# Index2Title + +#####——0 +prompt = {} +prompt["instruction"] = "What is the title of item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "What title is assigned to item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "Could you please tell me what item {item} is called?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Can you provide the title of item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "What item {item} is referred to as?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Would you mind informing me about the title of item {item}?" +prompt["response"] = "{title}" +index2item_prompt.append(prompt) + +# ======================================================== +# Index2Description + +#####——6 +prompt = {} +prompt["instruction"] = "Please provide a description of item {item}." +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "Briefly describe item {item}." +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "Can you share with me the description corresponding to item {item}?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "What is the description of item {item}?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "How to describe the characteristics of item {item}?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "Could you please tell me what item {item} looks like?" +prompt["response"] = "{description}" +index2item_prompt.append(prompt) + + +# ======================================================== +# index to Title and Description + +#####——12 +prompt = {} +prompt["instruction"] = "What is the title and description of item {item}?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——13 +prompt = {} +prompt["instruction"] = "Can you provide the corresponding title and description for item {item}?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——14 +prompt = {} +prompt["instruction"] = "Please tell me what item {item} is called, along with a brief description of it." +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——15 +prompt = {} +prompt["instruction"] = "Would you mind informing me about the title of the item {item} and how to describe its characteristics?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +#####——16 +prompt = {} +prompt["instruction"] = "I need to know the title and description of item {item}. Could you help me with that?" +prompt["response"] = "{title}\n\n{description}" +index2item_prompt.append(prompt) + +all_prompt["index2item"] = index2item_prompt + + + + + +# ======================================================== +# Task 4 -- FusionSequentialRec -- Prompt +# ======================================================== + + +fusionseqrec_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "The user has sequentially interacted with items {inters}. Can you recommend the next item for him? Tell me the title of the item？" +prompt["response"] = "{title}" +fusionseqrec_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "Based on the user's historical interactions: {inters}, try to predict the title of the item that the user may need next." +prompt["response"] = "{title}" +fusionseqrec_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "Utilizing the user's past ordered interactions, which include items {inters}, please recommend the next item you think is suitable for the user and provide its title." +prompt["response"] = "{title}" +fusionseqrec_prompt.append(prompt) + + +#####——3 +prompt = {} +prompt["instruction"] = "After interacting with items {inters}, what is the most probable item for the user to interact with next? Kindly provide the item's title." +prompt["response"] = "{title}" +fusionseqrec_prompt.append(prompt) + + + + + +#####——4 +prompt = {} +prompt["instruction"] = "Please review the user's historical interactions: {inters}, and describe what kind of item he still needs." +prompt["response"] = "{description}" +fusionseqrec_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Here is the item interaction history of the user: {inters}, please tell me what features he expects from his next item." +prompt["response"] = "{description}" +fusionseqrec_prompt.append(prompt) + +#####——6 +prompt = {} +prompt["instruction"] = "By analyzing the user's historical interactions with items {inters}, can you infer what the user's next interactive item will look like?" +prompt["response"] = "{description}" +fusionseqrec_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "Access the user's historical item interaction records: {inters}. Your objective is to describe the next potential item for him, taking into account his past interactions." +prompt["response"] = "{description}" +fusionseqrec_prompt.append(prompt) + + + + + + +#####——8 +prompt = {} +prompt["instruction"] = "Given the title sequence of user historical interactive items: {inter_titles}, can you recommend a suitable next item for the user?" +prompt["response"] = "{item}" +fusionseqrec_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "I possess a user's past interaction history, denoted by the title sequence of interactive items: {inter_titles}, and I am interested in knowing the user's next most desired item. Can you help me?" +prompt["response"] = "{item}" +fusionseqrec_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "Considering the title sequence of user history interaction items: {inter_titles}. What is the next recommendation for the user?" +prompt["response"] = "{item}" +fusionseqrec_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "You have obtained the ordered title list of user historical interaction items, as follows: {inter_titles}. Based on this historical context, kindly choose the subsequent item for user recommendation." +prompt["response"] = "{item}" +fusionseqrec_prompt.append(prompt) + + +all_prompt["fusionseqrec"] = fusionseqrec_prompt + + + + + + + +# ======================================================== +# Task 5 -- ItemSearch -- Prompt +# ======================================================== + + +itemsearch_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "Here is the historical interactions of a user: {inters}. And his personalized preferences are as follows: \"{explicit_preference}\". Your task is to recommend an item that is consistent with the user's preference." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "The user has interacted with a list of items, which are as follows: {inters}. Based on these interacted items, the user current intent is as follows \"{user_related_intention}\", and your task is to generate an item that matches the user's current intent." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——2 +prompt = {} +prompt["instruction"] = "As a recommender system, you are assisting a user who has recently interacted with the following items: {inters}. The user expresses a desire to obtain another item with the following characteristics: \"{item_related_intention}\". Please recommend an item that meets these criteria." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Using the user's current query: \"{query}\" and his historical interactions: {inters}, you can estimate the user's preferences \"{explicit_preference}\". Please respond to the user's query by selecting an item that best matches his preference and query." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "The user needs a new item and searches for: \"{query}\". In addition, he has previously interacted with: {inters}. You can obtain his preference by analyzing his historical interactions: \"{explicit_preference}\". Can you recommend an item that best matches the search query and preferences?" +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Based on the user's historical interactions with the following items: {inters}. You can infer his preference by observing the historical interactions: \"{explicit_preference}\". Now the user wants a new item and searches for: \"{query}\". Please select a suitable item that matches his preference and search intent." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + + + + + +#####——6 +prompt = {} +prompt["instruction"] = "Suppose you are a search engine, now a user searches that: \"{query}\", can you select an item to respond to the user's query?" +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "As a search engine, your task is to answer the user's query by generating a related item. The user's query is provided as \"{query}\". Please provide your generated item as your answer." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "As a recommender system, your task is to recommend an item that is related to the user's request, which is specified as follows: \"{query}\". Please provide your recommendation." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "You meet a user's query: \"{query}\". Please respond to this user by selecting an appropriate item." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + + +#####——10 +prompt = {} +prompt["instruction"] = "Your task is to recommend the best item that matches the user's query. Here is the search query of the user: \"{query}\", tell me the item you recommend." +prompt["response"] = "{item}" +itemsearch_prompt.append(prompt) + +all_prompt["itemsearch"] = itemsearch_prompt + + + + + +# ======================================================== +# Task 6 -- PreferenceObtain -- Prompt +# ======================================================== + +preferenceobtain_prompt = [] + +#####——0 +prompt = {} +prompt["instruction"] = "The user has interacted with items {inters} in chronological order. Please estimate his preferences." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——1 +prompt = {} +prompt["instruction"] = "Based on the items that the user has interacted with: {inters}, can you infer what preferences he has?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——3 +prompt = {} +prompt["instruction"] = "Can you provide a summary of the user's preferences based on his historical interactions: {inters}?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——4 +prompt = {} +prompt["instruction"] = "After interacting with items {inters} in order, what preferences do you think the user has?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——5 +prompt = {} +prompt["instruction"] = "Here is the item interaction history of the user: {inters}, could you please infer the user's preferences." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——6 +prompt = {} +prompt["instruction"] = "Based on the user's historical interaction records: {inters}, what are your speculations about his preferences?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——7 +prompt = {} +prompt["instruction"] = "Given the user's historical interactive items arranged in chronological order: {inters}, what can be inferred about the preferences of the user?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——8 +prompt = {} +prompt["instruction"] = "Can you speculate on the user's preferences based on his historical item interaction records: {inters}?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——9 +prompt = {} +prompt["instruction"] = "What is the preferences of a user who has previously interacted with items {inters} sequentially?" +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——10 +prompt = {} +prompt["instruction"] = "Using the user's historical interactions as input data, summarize the user's preferences. The historical interactions are provided as follows: {inters}." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +#####——11 +prompt = {} +prompt["instruction"] = "Utilizing the ordered list of the user's historical interaction items as a reference, please make an informed estimation of the user's preferences. The historical interactions are as follows: {inters}." +prompt["response"] = "{explicit_preference}" +preferenceobtain_prompt.append(prompt) + +all_prompt["preferenceobtain"] = preferenceobtain_prompt diff --git a/rq_llama.py b/rq_llama.py new file mode 100644 index 0000000000000000000000000000000000000000..859b5cf5f1e4edd2c238a1e9606fa29d8558aea9 --- /dev/null +++ b/rq_llama.py @@ -0,0 +1,261 @@ +import os +import json +import copy +import wandb +import torch +import torch.nn as nn +import transformers +from transformers import LlamaPreTrainedModel, LlamaForCausalLM, LlamaTokenizer, LlamaConfig + +from peft import ( + TaskType, + LoraConfig, + get_peft_model, + get_peft_model_state_dict, + set_peft_model_state_dict, +) + +from index.models import * +from index.models.rqvae import RQVAE + +from torch.nn.init import xavier_normal_ +from sklearn.cluster import KMeans + +class LlamaWithRQ(LlamaPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + args = config.args + + tokenizer = LlamaTokenizer.from_pretrained( + args['base_model'], + model_max_length = args['model_max_length'], + padding_side ="right", + ) + tokenizer.pad_token_id = 0 + + new_tokens = [] + prefix = ['','','','',''] + for i in range(len(args['num_emb_list'])): + new_tokens.extend([prefix[i].format(int(x)) for x in range(args['num_emb_list'][i])]) + self.prefix = prefix + + tokenizer.add_tokens(new_tokens) + config.vocab_size = len(tokenizer) + + llama_model = LlamaForCausalLM.from_pretrained(args['base_model']) + llama_model.resize_token_embeddings(len(tokenizer)) + + lora_config = LoraConfig( + r = args['lora_r'], + lora_alpha = args['lora_alpha'], + target_modules = args['lora_target_modules'].split(","), + modules_to_save = args['lora_modules_to_save'].split(","), + lora_dropout = args['lora_dropout'], + bias = "none", + inference_mode = False, + task_type = TaskType.CAUSAL_LM + ) + llama_model = get_peft_model(llama_model, lora_config) + + for n, p in llama_model.named_parameters(): + if "original_module" in n and any(module_name in n for module_name in lora_config.modules_to_save): + p.requires_grad = False + + self.tokenizer = tokenizer + self.model = llama_model + + item_json = os.path.join(args['data_path'], args['dataset'], args['dataset'] + ".item.json") + with open(item_json, 'r') as f: + self.item_texts = json.load(f) + + self.rqvae = RQVAE(in_dim = config.hidden_size, + num_emb_list = args['num_emb_list'], + e_dim = args['e_dim'], + layers = args['layers'], + dropout_prob = args['dropout_prob'], + bn = args['bn'], + loss_type = args['loss_type'], + quant_loss_weight = args['quant_loss_weight'], + kmeans_init = args['kmeans_init'], + kmeans_iters = args['kmeans_iters'], + sk_epsilons = args['sk_epsilons'], + sk_iters = args['sk_iters']) + + # self.projector = nn.Linear(args['e_dim'], config.hidden_size) + self.args = args + + def rqvae_forward(self, inputs, targets, inters, item, task): + llama_model = self.model.get_decoder() + if task.lower() in ['seqrec', 'itemsearch']: + # inputs, targets, inters, item + + # item-id to text + inter_feature_list = [] + inter_emb_list = [] + inter_item_list = inters.split(',') + for j in range(len(inter_item_list)): + inter_feature = self.item_texts[inter_item_list[j]]['title'] + ' ' + self.item_texts[inter_item_list[j]]['description'] + inter_id = self.tokenizer(inter_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) + inter_emb = llama_model(input_ids = inter_id.input_ids, attention_mask = inter_id.attention_mask) + inter_emb = inter_emb.last_hidden_state * inter_id.attention_mask.unsqueeze(-1) + inter_emb = inter_emb.sum(dim=1) / inter_id.attention_mask.sum(dim = -1, keepdim = True) + inter_emb_list.append(inter_emb.detach()) + inter_embs = torch.cat(inter_emb_list, dim = 0) + + item_feature = self.item_texts[item]['title'] + ' ' + self.item_texts[item]['description'] + item_ids = self.tokenizer(item_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) + item_emb = llama_model(input_ids = item_ids.input_ids, attention_mask = item_ids.attention_mask) + item_emb = item_emb.last_hidden_state * item_ids.attention_mask.unsqueeze(-1) + item_emb = item_emb.sum(dim=1) / item_ids.attention_mask.sum(dim = -1, keepdim = True) + item_emb = item_emb.detach() + + rec_embs, rq_loss, rqids = self.rqvae(torch.cat([inter_embs, item_emb], dim = 0)) + rqvae_loss, rec_loss = self.rqvae.compute_loss(rec_embs, rq_loss, torch.cat([inter_embs, item_emb], dim = 0)) + + inters_rqids = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist()[:-1] + item_rqid = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist()[-1] + + text_rqids = {} + code = '' + for rqid in inters_rqids: + for k, idx in enumerate(rqid): + code = code + self.prefix[k].format(idx) + code = code + ', ' + text_rqids['inters'] = code[:-2] + code = '' + for k, idx in enumerate(item_rqid): + code = code + self.prefix[k].format(idx) + text_rqids['item'] = code + + inputs = inputs.format(inters = text_rqids['inters']) + targets = targets.format(inters = text_rqids['inters'], item = text_rqids['item']) + + elif task.lower() in ['inters2title','inters2description','preferenceobtain']: + # inputs, targets, inters + inter_feature_list = [] + inter_emb_list = [] + inter_item_list = inters.split(',') + for j in range(len(inter_item_list)): + inter_feature = self.item_texts[inter_item_list[j]]['title'] + ' ' + self.item_texts[inter_item_list[j]]['description'] + inter_id = self.tokenizer(inter_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) + inter_emb = llama_model(input_ids = inter_id.input_ids, attention_mask = inter_id.attention_mask) + inter_emb = inter_emb.last_hidden_state * inter_id.attention_mask.unsqueeze(-1) + inter_emb = inter_emb.sum(dim=1) / inter_id.attention_mask.sum(dim = -1, keepdim = True) + inter_emb_list.append(inter_emb.detach()) + inter_embs = torch.cat(inter_emb_list, dim = 0) + + rec_embs, rq_loss, rqids = self.rqvae(inter_embs) + rqvae_loss, rec_loss = self.rqvae.compute_loss(rec_embs, rq_loss, inter_embs) + + inters_rqids = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist() + code = '' + for rqid in inters_rqids: + for k, idx in enumerate(rqid): + code = code + self.prefix[k].format(idx) + code = code + ', ' + + inputs = inputs.format(inters = code[:-2]) + targets = targets.format(inters = code[:-2]) + + elif task.lower() in ['item2index','index2item','intertitles2item','query2item']: + # inputs, targets, item + item_feature = self.item_texts[item]['title'] + ' ' + self.item_texts[item]['description'] + item_ids = self.tokenizer(item_feature, return_tensors = 'pt', padding=True, truncation=True).to(self.model.device) + item_emb = llama_model(input_ids = item_ids.input_ids, attention_mask = item_ids.attention_mask) + item_emb = item_emb.last_hidden_state * item_ids.attention_mask.unsqueeze(-1) + item_emb = item_emb.sum(dim=1) / item_ids.attention_mask.sum(dim = -1, keepdim = True) + item_emb = item_emb.detach() + + rec_embs, rq_loss, rqids = self.rqvae(item_emb) + rqvae_loss, rec_loss = self.rqvae.compute_loss(rec_embs, rq_loss, item_emb) + + item_rqid = rqids.view(-1, rqids.shape[-1]).cpu().numpy().tolist()[0] + code = '' + for k, idx in enumerate(item_rqid): + code = code + self.prefix[k].format(idx) + + targets = targets.format(item = code) + else: + raise NotImplementedError + + return inputs, targets, rqvae_loss, rec_embs.shape[0] + + def forward(self, input_ids, labels, inters, item, task): + ''' + 'input_ids': + [ + "Below is an instruction that describes a task. Write a response that appropriately completes the request. + ### Instruction: + Using the user's historical interactions as input data, suggest the next item that the user is highly likely to enjoy. + The historical interactions are provided as follows: {inters}. + ### Response:", + + 'Below is an instruction that describes a task. Write a response that appropriately completes the request. + ### Instruction: + You have obtained the ordered list of user historical interaction items, which is as follows: {inters}. + Using this history as a reference, please select the next item to recommend to the user. + ### Response:' + ], + + 'labels': + [ + "Below is an instruction that describes a task. Write a response that appropriately completes the request. + ### Instruction: + Using the user's historical interactions as input data, suggest the next item that the user is highly likely to enjoy. + The historical interactions are provided as follows: {inters}. + ### Response:{item}", + + 'Below is an instruction that describes a task. Write a response that appropriately completes the request. + ### Instruction: + You have obtained the ordered list of user historical interaction items, which is as follows: {inters}. + Using this history as a reference, please select the next item to recommend to the user. + ### Response:{item}' + ], + + 'inters': ['0', '0,1'], + 'item': ['1', '2'], + 'task': ['seqrec', 'seqrec'] + ''' + assert len(set([len(input_ids), len(labels), len(inters), len(item), len(task)])) == 1 + num_data = len(task) + + total_rqvae_loss = 0 + total_num_sample = 0 + for i in range(num_data): + input_ids[i], labels[i], rqvae_loss, num_sample = self.rqvae_forward(input_ids[i], labels[i], inters[i], item[i], task[i]) + total_rqvae_loss += rqvae_loss + total_num_sample += num_sample + + input_data = self.tokenizer( + text = labels, + text_target = input_ids, + return_tensors = 'pt', + padding = 'longest', + truncation = True, + max_length = self.tokenizer.model_max_length, + return_attention_mask = True + ).to(self.model.device) + + labels = copy.deepcopy(input_data["input_ids"]) + if self.args['only_train_response']: + labels[labels == self.tokenizer.pad_token_id] = -100 + labels[torch.where(input_data["labels"] != self.tokenizer.pad_token_id)] = -100 + + input_data["labels"] = labels + + # codebook_embedding = [] + # for i in range(len(self.rqvae.num_emb_list)): + # codebook_embedding.append(self.rqvae.rq.vq_layers[i].embedding.weight.data) + # codebook_embedding = torch.cat(codebook_embedding, dim = 0) + # codebook_embedding = self.projector(codebook_embedding) + # self.model.model.model.embed_tokens.weight.data[-codebook_embedding.shape[0]:] = codebook_embedding + + result = self.model(**input_data) + wandb.log({'Llama_Loss': result.loss, 'RQVAE_Loss': total_rqvae_loss / total_num_sample}) + result.loss += total_rqvae_loss / total_num_sample + wandb.log({'Total_Loss': result.loss}) + return result + + def floating_point_ops(self, inputs): + return 0 \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..03b9c6f6279c82ce8e169a1d971ff19dad42698c --- /dev/null +++ b/run.sh @@ -0,0 +1,98 @@ +export WANDB_MODE=disabled +export CUDA_LAUNCH_BLOCKING=1 + +DATASET=Games +BASE_MODEL= huggyllama/llama-7b +DATA_PATH=./data +OUTPUT_DIR=./ckpt/$DATASET/ + +torchrun --nproc_per_node=8 --master_port=3324 finetune.py \ + --base_model $BASE_MODEL \ + --output_dir $OUTPUT_DIR \ + --dataset $DATASET \ + --data_path $DATA_PATH \ + --per_device_batch_size 8 \ + --gradient_accumulation_steps 2 \ + --learning_rate 5e-5 \ + --epochs 4 \ + --weight_decay 0.01 \ + --save_and_eval_strategy epoch \ + --deepspeed ./config/ds_z3_bf16.json \ + --bf16 \ + --only_train_response \ + --tasks seqrec,item2index,index2item,fusionseqrec,itemsearch,preferenceobtain \ + --train_prompt_sample_num 1,1,1,1,1,1 \ + --train_data_sample_num 0,0,0,100000,0,0 \ + --index_file .index.json + + +cd convert +nohup ./convert.sh $OUTPUT_DIR >convert.log 2>&1 & +cd .. + + + + + + +DATASET=Arts +BASE_MODEL= huggyllama/llama-7b +DATA_PATH=./data +OUTPUT_DIR=./ckpt/$DATASET/ + +torchrun --nproc_per_node=8 --master_port=3324 finetune.py \ + --base_model $BASE_MODEL \ + --output_dir $OUTPUT_DIR \ + --dataset $DATASET \ + --data_path $DATA_PATH \ + --per_device_batch_size 8 \ + --gradient_accumulation_steps 2 \ + --learning_rate 5e-5 \ + --epochs 4 \ + --weight_decay 0.01 \ + --save_and_eval_strategy epoch \ + --deepspeed ./config/ds_z3_bf16.json \ + --bf16 \ + --only_train_response \ + --tasks seqrec,item2index,index2item,fusionseqrec,itemsearch,preferenceobtain \ + --train_prompt_sample_num 1,1,1,1,1,1 \ + --train_data_sample_num 0,0,0,30000,0,0 \ + --index_file .index.json + + +cd convert +nohup ./convert.sh $OUTPUT_DIR >convert.log 2>&1 & +cd .. + + + + + +DATASET=Instruments +BASE_MODEL= huggyllama/llama-7b +DATA_PATH=./data +OUTPUT_DIR=./ckpt/$DATASET/ + +torchrun --nproc_per_node=8 --master_port=3324 finetune.py \ + --base_model $BASE_MODEL \ + --output_dir $OUTPUT_DIR \ + --dataset $DATASET \ + --data_path $DATA_PATH \ + --per_device_batch_size 8 \ + --gradient_accumulation_steps 2 \ + --learning_rate 5e-5 \ + --epochs 4 \ + --weight_decay 0.01 \ + --save_and_eval_strategy epoch \ + --deepspeed ./config/ds_z3_bf16.json \ + --bf16 \ + --only_train_response \ + --tasks seqrec,item2index,index2item,fusionseqrec,itemsearch,preferenceobtain \ + --train_prompt_sample_num 1,1,1,1,1,1 \ + --train_data_sample_num 0,0,0,20000,0,0 \ + --index_file .index.json + + +cd convert +nohup ./convert.sh $OUTPUT_DIR >convert.log 2>&1 & +cd .. diff --git a/run_test_ddp.sh b/run_test_ddp.sh new file mode 100644 index 0000000000000000000000000000000000000000..efa9459202985bf9460af1a71d82cced6001c816 --- /dev/null +++ b/run_test_ddp.sh @@ -0,0 +1,15 @@ +DATASET=Instruments +DATA_PATH=${datain}/v-yinju/rqvae-zzx/data +CKPT_PATH=${datain}/v-yinju/rq-llama/v3-train/$DATASET/second/finetune +RESULTS_FILE=$CKPT_PATH/eval_result.json + +torchrun --nproc_per_node=2 --master_port=4324 evaluate-finetuned.py \ + --ckpt_path $CKPT_PATH \ + --dataset $DATASET \ + --data_path $DATA_PATH \ + --results_file $RESULTS_FILE \ + --test_batch_size 1 \ + --num_beams 20 \ + --test_prompt_ids all \ + --test_task seqrec \ + --index_file $CKPT_PATH/indices.json \ No newline at end of file diff --git a/test-main.py b/test-main.py new file mode 100644 index 0000000000000000000000000000000000000000..60a38fbe849ccd382d945a92eabf67564fa9438e --- /dev/null +++ b/test-main.py @@ -0,0 +1,184 @@ +import argparse +import json +import os +import sys + +import torch +import transformers +import torch.distributed as dist +from torch.utils.data.distributed import DistributedSampler +from torch.nn.parallel import DistributedDataParallel +from peft import PeftModel +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig + +from utils import * +from collator import VanillaCollator, TestCollator +from prompt import all_prompt +from evaluate import get_topk_results, get_metrics_results +from rq_llama import * + +parser = argparse.ArgumentParser(description="RQ-Llama Evaluation") +parser = parse_global_args(parser) +parser = parse_dataset_args(parser) +parser = parse_test_args(parser) +args = parser.parse_args() + +set_seed(args.seed) +world_size = int(os.environ.get("WORLD_SIZE", 1)) +local_rank = int(os.environ.get("LOCAL_RANK") or 0) +torch.cuda.set_device(local_rank) +if local_rank == 0: + print(vars(args)) + +dist.init_process_group(backend = "nccl", world_size = world_size, rank = local_rank) + +device_map = {"": local_rank} +device = torch.device("cuda",local_rank) + +rqllama = LlamaWithRQ.from_pretrained(args.ckpt_path, torch_dtype = torch.float16, low_cpu_mem_usage = True, device_map = device_map) +rqllama = DistributedDataParallel(rqllama, device_ids = [local_rank]) + +if args.test_prompt_ids == "all": + if args.test_task.lower() == "seqrec": + prompt_ids = range(len(all_prompt["seqrec"])) + elif args.test_task.lower() == "itemsearch": + prompt_ids = range(len(all_prompt["itemsearch"])) + elif args.test_task.lower() == "fusionseqrec": + prompt_ids = range(len(all_prompt["fusionseqrec"])) +else: + prompt_ids = [int(_) for _ in args.test_prompt_ids.split(",")] + +test_data = load_test_dataset(args) +if local_rank == 0: + print("data num:", len(test_data)) +ddp_sampler = DistributedSampler(test_data, num_replicas = world_size, rank = local_rank, drop_last = True) +collator = TestCollator(args, rqllama.module.tokenizer) +all_items = test_data.get_all_items() +# print('num_items:', len(all_items)) +prefix_allowed_tokens = test_data.get_prefix_allowed_tokens_fn(rqllama.module.tokenizer) + +test_loader = DataLoader( + test_data, + batch_size = args.test_batch_size, + collate_fn = collator, + sampler = ddp_sampler, + num_workers = 2, + pin_memory = True +) + +rqllama.eval() + +metrics = args.metrics.split(",") +all_prompt_results = [] + +with torch.no_grad(): + for prompt_id in prompt_ids: + if local_rank == 0: + print("Start prompt: ",prompt_id) + + test_loader.dataset.set_prompt(prompt_id) + metrics_results = {} + total = 0 + for step, batch in enumerate(tqdm(test_loader)): + inputs = batch[0].to(device) + targets = batch[1] + bs = len(targets) + num_beams = args.num_beams + + while True: + try: + output = rqllama.module.model.generate( + input_ids = inputs["input_ids"], + attention_mask = inputs["attention_mask"], + max_new_tokens = 10, + prefix_allowed_tokens_fn = prefix_allowed_tokens, + num_beams = num_beams, + num_return_sequences = num_beams, + output_scores = True, + return_dict_in_generate = True, + early_stopping = True, + ) + break + except torch.cuda.OutOfMemoryError as e: + print("Out of memory!") + num_beams = num_beams -1 + print("Beam:", num_beams) + except Exception: + raise RuntimeError + + output_ids = output["sequences"] + scores = output["sequences_scores"] + + output = rqllama.module.tokenizer.batch_decode(output_ids, skip_special_tokens = True) + topk_res = get_topk_results( + output, + scores, + targets, + num_beams, + all_items = all_items if args.filter_items else None + ) + + bs_gather_list = [None for _ in range(world_size)] + dist.all_gather_object(obj = bs, object_list = bs_gather_list) + total += sum(bs_gather_list) + res_gather_list = [None for _ in range(world_size)] + dist.all_gather_object(obj = topk_res, object_list = res_gather_list) + + if local_rank == 0: + all_device_topk_res = [] + for ga_res in res_gather_list: + all_device_topk_res += ga_res + batch_metrics_res = get_metrics_results(all_device_topk_res, metrics) + for m, res in batch_metrics_res.items(): + if m not in metrics_results: + metrics_results[m] = res + else: + metrics_results[m] += res + if (step + 1) % 50 == 0: + temp = {} + for m in metrics_results: + temp[m] = metrics_results[m] / total + print(temp) + dist.barrier() + + if local_rank == 0: + for m in metrics_results: + metrics_results[m] = metrics_results[m] / total + + all_prompt_results.append(metrics_results) + print("======================================================") + print("Prompt {} results: ".format(prompt_id), metrics_results) + print("======================================================") + print("") + dist.barrier() +dist.barrier() + +if local_rank == 0: + mean_results = {} + min_results = {} + max_results = {} + + for m in metrics: + all_res = [_[m] for _ in all_prompt_results] + mean_results[m] = sum(all_res) / len(all_res) + min_results[m] = min(all_res) + max_results[m] = max(all_res) + + print("======================================================") + print("Mean results: ", mean_results) + print("Min results: ", min_results) + print("Max results: ", max_results) + print("======================================================") + + save_data={} + save_data["test_prompt_ids"] = args.test_prompt_ids + save_data["mean_results"] = mean_results + save_data["min_results"] = min_results + save_data["max_results"] = max_results + save_data["all_prompt_results"] = all_prompt_results + + with open(args.results_file, "w") as f: + json.dump(save_data, f, indent = 4) + print("Save file: ", args.results_file) \ No newline at end of file diff --git a/utils.py b/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6d127013e6fc25b1130b863b7bde650779a5713c --- /dev/null +++ b/utils.py @@ -0,0 +1,436 @@ +import json +import logging +import os +import random +import datetime + +import numpy as np +import torch +from torch.utils.data import ConcatDataset +from data import SeqRecDataset, ItemFeatDataset, ItemSearchDataset, FusionSeqRecDataset, SeqRecTestDataset, PreferenceObtainDataset +from data_finetune import SeqRecFinetune, ItemFeatFinetune, ItemSearchFinetune, FusionSeqRecFinetune, PreferenceObtainFinetune + +def parse_evaluate_args(parser): + parser.add_argument("--seed", type=int, default=42, help="Random seed") + parser.add_argument("--base_model", type=str, default="../llama-7b/", help="basic model path") + parser.add_argument("--output_dir", type=str, default="./ckpt/", help="The output directory") + + parser.add_argument("--data_path", type=str, default="", + help="data directory") + parser.add_argument("--tasks", type=str, + default='seqrec,itemsearch,inters2title,inters2description,preferenceobtain,item2index,index2item,intertitles2item,query2item', + help="Downstream tasks, separate by comma") + parser.add_argument("--train_data_sample_num", type=str, default="0,0,0,0,0,0,0,0,0", + help="the number of sampling data for each task") + parser.add_argument("--dataset", type=str, default="Instruments", help="Dataset name") + parser.add_argument("--index_file", type=str, default=".index.json", help="the item indices file") + parser.add_argument("--dataloader_num_workers", type=int, default=0, help="dataloader num_workers") + parser.add_argument("--dataloader_prefetch_factor", type=int, default=2, help="dataloader prefetch_factor") + + # arguments related to sequential task + parser.add_argument("--max_his_len", type=int, default=20, + help="the max number of items in history sequence, -1 means no limit") + parser.add_argument("--add_prefix", action="store_true", default=False, + help="whether add sequential prefix in history") + parser.add_argument("--his_sep", type=str, default=", ", help="The separator used for history") + parser.add_argument("--only_train_response", action="store_true", default=False, + help="whether only train on responses") + + parser.add_argument("--train_prompt_sample_num", type=str, default="1,1,1,1,1,1,1,1,1", + help="the number of sampling prompts for each task") + + parser.add_argument("--valid_prompt_id", type=int, default=0, + help="The prompt used for validation") + parser.add_argument("--sample_valid", action="store_true", default=True, + help="use sampled prompt for validation") + parser.add_argument("--valid_prompt_sample_num", type=int, default=2, + help="the number of sampling validation sequential recommendation prompts") + + parser.add_argument("--ckpt_path", type=str, default="", help="The checkpoint path") + parser.add_argument("--lora", action="store_true", default=False) + parser.add_argument("--filter_items", action="store_true", default=False, + help="whether filter illegal items") + + parser.add_argument("--results_file", type=str, default="./results/test-ddp.json", help="result output path") + + parser.add_argument("--test_batch_size", type=int, default=1) + parser.add_argument("--num_beams", type=int, default=20) + parser.add_argument("--sample_num", type=int, default=-1, + help="test sample number, -1 represents using all test data") + parser.add_argument("--gpu_id", type=int, default=0, + help="GPU ID when testing with single GPU") + parser.add_argument("--test_prompt_ids", type=str, default="0", + help="test prompt ids, separate by comma. 'all' represents using all") + parser.add_argument("--metrics", type=str, default="hit@1,hit@5,hit@10,ndcg@5,ndcg@10", + help="test metrics, separate by comma") + parser.add_argument("--test_task", type=str, default="SeqRec") + + return parser + +def parse_finetune_args(parser): + parser.add_argument("--seed", type=int, default=42, help="Random seed") + parser.add_argument("--base_model", type=str, default="../llama-7b/", help="basic model path") + + parser.add_argument("--output_dir", type=str, default="./ckpt/", help="The output directory") + + parser.add_argument("--data_path", type=str, default="", + help="data directory") + parser.add_argument("--tasks", type=str, + default='seqrec,itemsearch,inters2title,inters2description,preferenceobtain,item2index,index2item,intertitles2item,query2item', + help="Downstream tasks, separate by comma") + parser.add_argument("--train_data_sample_num", type=str, default="0,0,0,0,0,0,0,0,0", + help="the number of sampling data for each task") + parser.add_argument("--dataset", type=str, default="Instruments", help="Dataset name") + parser.add_argument("--index_file", type=str, default=".index.json", help="the item indices file") + parser.add_argument("--dataloader_num_workers", type=int, default=0, help="dataloader num_workers") + parser.add_argument("--dataloader_prefetch_factor", type=int, default=2, help="dataloader prefetch_factor") + + parser.add_argument("--max_his_len", type=int, default=20, + help="the max number of items in history sequence, -1 means no limit") + parser.add_argument("--add_prefix", action="store_true", default=False, + help="whether add sequential prefix in history") + parser.add_argument("--his_sep", type=str, default=", ", help="The separator used for history") + parser.add_argument("--only_train_response", action="store_true", default=False, + help="whether only train on responses") + + parser.add_argument("--train_prompt_sample_num", type=str, default="1,1,1,1,1,1,1,1,1", + help="the number of sampling prompts for each task") + + parser.add_argument("--valid_prompt_id", type=int, default=0, + help="The prompt used for validation") + parser.add_argument("--sample_valid", action="store_true", default=True, + help="use sampled prompt for validation") + parser.add_argument("--valid_prompt_sample_num", type=int, default=2, + help="the number of sampling validation sequential recommendation prompts") + + parser.add_argument("--optim", type=str, default="adamw_torch", help='The name of the optimizer') + parser.add_argument("--epochs", type=int, default=4) + parser.add_argument("--learning_rate", type=float, default=2e-5) + parser.add_argument("--per_device_batch_size", type=int, default=8) + parser.add_argument("--gradient_accumulation_steps", type=int, default=2) + parser.add_argument("--logging_step", type=int, default=10) + parser.add_argument("--model_max_length", type=int, default=2048) + parser.add_argument("--weight_decay", type=float, default=0.01) + + parser.add_argument("--lora_r", type=int, default=8) + parser.add_argument("--lora_alpha", type=int, default=32) + parser.add_argument("--lora_dropout", type=float, default=0.05) + parser.add_argument("--lora_target_modules", type=str, + default="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj", help="separate by comma") + parser.add_argument("--lora_modules_to_save", type=str, + default="embed_tokens,lm_head", help="separate by comma") + + parser.add_argument("--resume_from_checkpoint", type=str, default=None, help="either training checkpoint or final adapter") + + parser.add_argument("--warmup_ratio", type=float, default=0.01) + parser.add_argument("--lr_scheduler_type", type=str, default="cosine") + parser.add_argument("--save_and_eval_strategy", type=str, default="epoch") + parser.add_argument("--save_and_eval_steps", type=int, default=1000) + parser.add_argument("--fp16", action="store_true", default=False) + parser.add_argument("--bf16", action="store_true", default=False) + parser.add_argument("--deepspeed", type=str, default="./config/ds_z3_bf16.json") + parser.add_argument("--remove_unused_columns", action="store_true", default=False, help='if remove unused columns') + + parser.add_argument("--re_index", type = int, default = 0) + parser.add_argument("--ckpt_path", type=str, default="") + + return parser + +def load_finetune_datasets(args): + + tasks = args.tasks.split(",") + + train_prompt_sample_num = [int(_) for _ in args.train_prompt_sample_num.split(",")] + assert len(tasks) == len(train_prompt_sample_num), "prompt sample number does not match task number" + train_data_sample_num = [int(_) for _ in args.train_data_sample_num.split(",")] + assert len(tasks) == len(train_data_sample_num), "data sample number does not match task number" + + train_datasets = [] + for task, prompt_sample_num,data_sample_num in zip(tasks,train_prompt_sample_num,train_data_sample_num): + if task.lower() == "seqrec": + dataset = SeqRecFinetune(args, mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() == "item2index" or task.lower() == "index2item": + dataset = ItemFeatFinetune(args, task=task.lower(), prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() == "fusionseqrec": + dataset = FusionSeqRecFinetune(args, mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() == "itemsearch": + dataset = ItemSearchFinetune(args, mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() == "preferenceobtain": + dataset = PreferenceObtainFinetune(args, prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + else: + raise NotImplementedError + train_datasets.append(dataset) + + train_data = ConcatDataset(train_datasets) + + valid_data = SeqRecFinetune(args, "valid", args.valid_prompt_sample_num) + + return train_data, valid_data + +# def load_finetune_datasets(args): +# tasks = args.tasks.split(",") +# train_prompt_sample_num = [int(_) for _ in args.train_prompt_sample_num.split(",")] +# assert len(tasks) == len(train_prompt_sample_num), "prompt sample number does not match task number" +# train_data_sample_num = [int(_) for _ in args.train_data_sample_num.split(",")] +# assert len(tasks) == len(train_data_sample_num), "data sample number does not match task number" + +# train_datasets = [] +# for task, prompt_sample_num,data_sample_num in zip(tasks,train_prompt_sample_num,train_data_sample_num): +# if task.lower() == "seqrec": +# dataset = SeqRecFinetune(args, mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + +# elif task.lower() == "item2index" or task.lower() == "index2item": +# dataset = ItemFeatFinetune(args, task=task.lower(), prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + +# elif task.lower() in ["inters2title", "inters2description", "intertitles2item"]: +# dataset = FusionSeqRecFinetune(args, task=task.lower(), mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + +# elif task.lower() in ["itemsearch", "query2item"]: +# dataset = ItemSearchFinetune(args, task=task.lower(),mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + +# elif task.lower() == "preferenceobtain": +# dataset = PreferenceObtainFinetune(args, prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + +# else: +# raise NotImplementedError +# train_datasets.append(dataset) + +# train_data = ConcatDataset(train_datasets) + +# valid_data = SeqRecDataset(args,"valid",args.valid_prompt_sample_num) + +# return train_data, valid_data + +def parse_global_args(parser): + parser.add_argument("--seed", type=int, default=42, help="Random seed") + + parser.add_argument("--base_model", type=str, + default="../llama-7b/", + help="basic model path") + parser.add_argument("--output_dir", type=str, + default="./ckpt/", + help="The output directory") + return parser + +def parse_dataset_args(parser): + parser.add_argument("--data_path", type=str, default="", + help="data directory") + parser.add_argument("--tasks", type=str, + default='seqrec,itemsearch,inters2title,inters2description,preferenceobtain,item2index,index2item,intertitles2item,query2item', + help="Downstream tasks, separate by comma") + parser.add_argument("--train_data_sample_num", type=str, default="0,0,0,0,0,0,0,0,0", + help="the number of sampling data for each task") + parser.add_argument("--dataset", type=str, default="Instruments", help="Dataset name") + parser.add_argument("--index_file", type=str, default=".index.json", help="the item indices file") + parser.add_argument("--dataloader_num_workers", type=int, default=0, help="dataloader num_workers") + parser.add_argument("--dataloader_prefetch_factor", type=int, default=2, help="dataloader prefetch_factor") + + # arguments related to sequential task + parser.add_argument("--max_his_len", type=int, default=20, + help="the max number of items in history sequence, -1 means no limit") + parser.add_argument("--add_prefix", action="store_true", default=False, + help="whether add sequential prefix in history") + parser.add_argument("--his_sep", type=str, default=", ", help="The separator used for history") + parser.add_argument("--only_train_response", action="store_true", default=False, + help="whether only train on responses") + + parser.add_argument("--train_prompt_sample_num", type=str, default="1,1,1,1,1,1,1,1,1", + help="the number of sampling prompts for each task") + + parser.add_argument("--valid_prompt_id", type=int, default=0, + help="The prompt used for validation") + parser.add_argument("--sample_valid", action="store_true", default=True, + help="use sampled prompt for validation") + parser.add_argument("--valid_prompt_sample_num", type=int, default=2, + help="the number of sampling validation sequential recommendation prompts") + + return parser + +def parse_train_args(parser): + parser.add_argument("--optim", type=str, default="adamw_torch", help='The name of the optimizer') + parser.add_argument("--epochs", type=int, default=4) + parser.add_argument("--learning_rate", type=float, default=2e-5) + parser.add_argument("--per_device_batch_size", type=int, default=8) + parser.add_argument("--gradient_accumulation_steps", type=int, default=2) + parser.add_argument("--logging_step", type=int, default=10) + parser.add_argument("--model_max_length", type=int, default=2048) + parser.add_argument("--weight_decay", type=float, default=0.01) + + parser.add_argument("--lora_r", type=int, default=8) + parser.add_argument("--lora_alpha", type=int, default=32) + parser.add_argument("--lora_dropout", type=float, default=0.05) + parser.add_argument("--lora_target_modules", type=str, + default="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj", help="separate by comma") + parser.add_argument("--lora_modules_to_save", type=str, + default="embed_tokens,lm_head", help="separate by comma") + + parser.add_argument("--resume_from_checkpoint", type=str, default=None, help="either training checkpoint or final adapter") + + parser.add_argument("--warmup_ratio", type=float, default=0.01) + parser.add_argument("--lr_scheduler_type", type=str, default="cosine") + parser.add_argument("--save_and_eval_strategy", type=str, default="epoch") + parser.add_argument("--save_and_eval_steps", type=int, default=1000) + parser.add_argument("--fp16", action="store_true", default=False) + parser.add_argument("--bf16", action="store_true", default=False) + parser.add_argument("--deepspeed", type=str, default="./config/ds_z3_bf16.json") + parser.add_argument("--remove_unused_columns", action="store_true", default=False, help='if remove unused columns') + + return parser + +def parse_rqvae_args(parser): + parser.add_argument('--lr', type=float, default=1e-3, help='learning rate') + # parser.add_argument('--epochs', type=int, default=5000, help='number of epochs') + parser.add_argument('--batch_size', type=int, default=1024, help='batch size') + parser.add_argument('--num_workers', type=int, default=4, ) + parser.add_argument('--eval_step', type=int, default=50, help='eval step') + parser.add_argument('--learner', type=str, default="AdamW", help='optimizer') + # parser.add_argument("--data_path", type=str, + # default="../data/Games/Games.emb-llama-td.npy", + # help="Input data path.") + + # parser.add_argument('--weight_decay', type=float, default=1e-4, help='l2 regularization weight') + parser.add_argument("--dropout_prob", type=float, default=0.0, help="dropout ratio") + parser.add_argument("--bn", type=bool, default=False, help="use bn or not") + parser.add_argument("--loss_type", type=str, default="mse", help="loss_type") + parser.add_argument("--kmeans_init", type=bool, default=False, help="use kmeans_init or not") + parser.add_argument("--kmeans_iters", type=int, default=100, help="max kmeans iters") + parser.add_argument('--sk_epsilons', type=float, nargs='+', default=[0.0, 0.0, 0.0, 0.0], help="sinkhorn epsilons") + parser.add_argument("--sk_iters", type=int, default=50, help="max sinkhorn iters") + + parser.add_argument("--device", type=str, default="cuda:1", help="gpu or cpu") + + parser.add_argument('--num_emb_list', type=int, nargs='+', default=[256,256,256,256], help='emb num of every vq') + parser.add_argument('--e_dim', type=int, default=32, help='vq codebook embedding size') + parser.add_argument('--quant_loss_weight', type=float, default=1.0, help='vq quantion loss weight') + parser.add_argument('--layers', type=int, nargs='+', default=[2048,1024,512,256,128,64], help='hidden sizes of every layer') + + parser.add_argument("--ckpt_dir", type=str, default="", help="output directory for model") + parser.add_argument("--warmup", type=int, default=5, help="epochs for warmup") + parser.add_argument("--rqvae_model", type=str, default="", help="") + + return parser + +def parse_pretrain_args(parser): + parser.add_argument("--ckpt_path", type=str, default="", help="output directory for model") + return parser + +def parse_test_args(parser): + + parser.add_argument("--ckpt_path", type=str, + default="", + help="The checkpoint path") + parser.add_argument("--lora", action="store_true", default=False) + parser.add_argument("--filter_items", action="store_true", default=False, + help="whether filter illegal items") + + parser.add_argument("--results_file", type=str, + default="./results/test-ddp.json", + help="result output path") + + parser.add_argument("--test_batch_size", type=int, default=1) + parser.add_argument("--num_beams", type=int, default=20) + parser.add_argument("--sample_num", type=int, default=-1, + help="test sample number, -1 represents using all test data") + parser.add_argument("--gpu_id", type=int, default=0, + help="GPU ID when testing with single GPU") + parser.add_argument("--test_prompt_ids", type=str, default="0", + help="test prompt ids, separate by comma. 'all' represents using all") + parser.add_argument("--metrics", type=str, default="hit@1,hit@5,hit@10,ndcg@5,ndcg@10", + help="test metrics, separate by comma") + parser.add_argument("--test_task", type=str, default="SeqRec") + + + return parser + +def get_local_time(): + cur = datetime.datetime.now() + cur = cur.strftime("%b-%d-%Y_%H-%M-%S") + + return cur + + +def set_seed(seed): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.enabled = False + +def ensure_dir(dir_path): + + os.makedirs(dir_path, exist_ok=True) + + +def load_datasets(args): + + tasks = args.tasks.split(",") + + train_prompt_sample_num = [int(_) for _ in args.train_prompt_sample_num.split(",")] + assert len(tasks) == len(train_prompt_sample_num), "prompt sample number does not match task number" + train_data_sample_num = [int(_) for _ in args.train_data_sample_num.split(",")] + assert len(tasks) == len(train_data_sample_num), "data sample number does not match task number" + + train_datasets = [] + for task, prompt_sample_num,data_sample_num in zip(tasks,train_prompt_sample_num,train_data_sample_num): + if task.lower() == "seqrec": + dataset = SeqRecDataset(args, mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() == "item2index" or task.lower() == "index2item": + dataset = ItemFeatDataset(args, task=task.lower(), prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() in ["inters2title", "inters2description", "intertitles2item"]: + dataset = FusionSeqRecDataset(args, task=task.lower(), mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() in ["itemsearch", "query2item"]: + dataset = ItemSearchDataset(args, task=task.lower(),mode="train", prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + elif task.lower() == "preferenceobtain": + dataset = PreferenceObtainDataset(args, prompt_sample_num=prompt_sample_num, sample_num=data_sample_num) + + else: + raise NotImplementedError + train_datasets.append(dataset) + + train_data = ConcatDataset(train_datasets) + + valid_data = SeqRecDataset(args,"valid",args.valid_prompt_sample_num) + + return train_data, valid_data + +def load_test_dataset(args): + + if args.test_task.lower() == "seqrec": + test_data = SeqRecFinetune(args, mode="test", sample_num=args.sample_num) + elif args.test_task.lower() == "itemsearch": + test_data = ItemSearchDataset(args, mode="test", sample_num=args.sample_num) + elif args.test_task.lower() == "fusionseqrec": + test_data = FusionSeqRecDataset(args, mode="test", sample_num=args.sample_num) + else: + raise NotImplementedError + + return test_data + +# def load_test_dataset(args): + +# if args.test_task.lower() == "seqrec": +# test_data = SeqRecDataset(args, mode="test", sample_num=args.sample_num) +# elif args.test_task.lower() == "itemsearch": +# test_data = ItemSearchDataset(args, mode="test", sample_num=args.sample_num) +# elif args.test_task.lower() == "fusionseqrec": +# test_data = FusionSeqRecDataset(args, mode="test", sample_num=args.sample_num) +# else: +# raise NotImplementedError + +# return test_data + +def load_json(file): + with open(file, 'r') as f: + data = json.load(f) + return data