File size: 143,486 Bytes
c15e89d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 | {
"architectures": [
"Qwen3_5MoeForConditionalGeneration"
],
"image_token_id": 248056,
"model_type": "qwen3_5_moe",
"text_config": {
"attention_bias": false,
"attention_dropout": 0.0,
"attn_output_gate": true,
"dtype": "bfloat16",
"eos_token_id": 248044,
"full_attention_interval": 4,
"head_dim": 256,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"layer_types": [
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"linear_attention",
"linear_attention",
"full_attention"
],
"linear_conv_kernel_dim": 4,
"linear_key_head_dim": 128,
"linear_num_key_heads": 16,
"linear_num_value_heads": 64,
"linear_value_head_dim": 128,
"max_position_embeddings": 262144,
"mlp_only_layers": [],
"model_type": "qwen3_5_moe_text",
"moe_intermediate_size": 1024,
"mtp_num_hidden_layers": 1,
"mtp_use_dedicated_embeddings": false,
"num_attention_heads": 32,
"num_experts": 512,
"num_experts_per_tok": 10,
"num_hidden_layers": 60,
"num_key_value_heads": 2,
"rms_norm_eps": 1e-06,
"router_aux_loss_coef": 0.001,
"shared_expert_intermediate_size": 1024,
"use_cache": true,
"vocab_size": 248320,
"mamba_ssm_dtype": "float32",
"rope_parameters": {
"mrope_interleaved": true,
"mrope_section": [
11,
11,
10
],
"rope_type": "default",
"rope_theta": 10000000,
"partial_rotary_factor": 0.25
}
},
"tie_word_embeddings": false,
"transformers_version": "4.57.0.dev0",
"video_token_id": 248057,
"vision_config": {
"deepstack_visual_indexes": [],
"depth": 27,
"hidden_act": "gelu_pytorch_tanh",
"hidden_size": 1152,
"in_channels": 3,
"initializer_range": 0.02,
"intermediate_size": 4304,
"model_type": "qwen3_5_moe",
"num_heads": 16,
"num_position_embeddings": 2304,
"out_hidden_size": 4096,
"patch_size": 16,
"spatial_merge_size": 2,
"temporal_patch_size": 2
},
"vision_end_token_id": 248054,
"vision_start_token_id": 248053,
"quantization_config": {
"config_groups": {
"group_0": {
"input_activations": {
"actorder": null,
"block_structure": null,
"dynamic": true,
"group_size": null,
"num_bits": 8,
"observer": null,
"observer_kwargs": {},
"strategy": "token",
"symmetric": true,
"type": "float"
},
"output_activations": null,
"targets": [
"Linear"
],
"weights": {
"actorder": null,
"block_structure": null,
"dynamic": false,
"group_size": null,
"num_bits": 8,
"observer": "minmax",
"observer_kwargs": {},
"strategy": "channel",
"symmetric": true,
"type": "float"
}
}
},
"format": "float-quantized",
"global_compression_ratio": null,
"ignore": [
"lm_head",
"model.language_model.embed_tokens",
"model.language_model.layers.0.input_layernorm",
"model.language_model.layers.0.linear_attn.A_log",
"model.language_model.layers.0.linear_attn.conv1d",
"model.language_model.layers.0.linear_attn.dt",
"model.language_model.layers.0.linear_attn.in_proj_a",
"model.language_model.layers.0.linear_attn.in_proj_b",
"model.language_model.layers.0.linear_attn.in_proj_qkv",
"model.language_model.layers.0.linear_attn.in_proj_z",
"model.language_model.layers.0.linear_attn.norm",
"model.language_model.layers.0.linear_attn.out_proj",
"model.language_model.layers.0.mlp.gate",
"model.language_model.layers.0.mlp.shared_expert_gate",
"model.language_model.layers.0.post_attention_layernorm",
"model.language_model.layers.1.input_layernorm",
"model.language_model.layers.1.linear_attn.A_log",
"model.language_model.layers.1.linear_attn.conv1d",
"model.language_model.layers.1.linear_attn.dt",
"model.language_model.layers.1.linear_attn.in_proj_a",
"model.language_model.layers.1.linear_attn.in_proj_b",
"model.language_model.layers.1.linear_attn.in_proj_qkv",
"model.language_model.layers.1.linear_attn.in_proj_z",
"model.language_model.layers.1.linear_attn.norm",
"model.language_model.layers.1.linear_attn.out_proj",
"model.language_model.layers.1.mlp.gate",
"model.language_model.layers.1.mlp.shared_expert_gate",
"model.language_model.layers.1.post_attention_layernorm",
"model.language_model.layers.10.input_layernorm",
"model.language_model.layers.10.linear_attn.A_log",
"model.language_model.layers.10.linear_attn.conv1d",
"model.language_model.layers.10.linear_attn.dt",
"model.language_model.layers.10.linear_attn.in_proj_a",
"model.language_model.layers.10.linear_attn.in_proj_b",
"model.language_model.layers.10.linear_attn.in_proj_qkv",
"model.language_model.layers.10.linear_attn.in_proj_z",
"model.language_model.layers.10.linear_attn.norm",
"model.language_model.layers.10.linear_attn.out_proj",
"model.language_model.layers.10.mlp.gate",
"model.language_model.layers.10.mlp.shared_expert_gate",
"model.language_model.layers.10.post_attention_layernorm",
"model.language_model.layers.11.input_layernorm",
"model.language_model.layers.11.mlp.gate",
"model.language_model.layers.11.mlp.shared_expert_gate",
"model.language_model.layers.11.post_attention_layernorm",
"model.language_model.layers.11.self_attn.k_norm",
"model.language_model.layers.11.self_attn.k_proj",
"model.language_model.layers.11.self_attn.o_proj",
"model.language_model.layers.11.self_attn.q_norm",
"model.language_model.layers.11.self_attn.q_proj",
"model.language_model.layers.11.self_attn.v_proj",
"model.language_model.layers.12.input_layernorm",
"model.language_model.layers.12.linear_attn.A_log",
"model.language_model.layers.12.linear_attn.conv1d",
"model.language_model.layers.12.linear_attn.dt",
"model.language_model.layers.12.linear_attn.in_proj_a",
"model.language_model.layers.12.linear_attn.in_proj_b",
"model.language_model.layers.12.linear_attn.in_proj_qkv",
"model.language_model.layers.12.linear_attn.in_proj_z",
"model.language_model.layers.12.linear_attn.norm",
"model.language_model.layers.12.linear_attn.out_proj",
"model.language_model.layers.12.mlp.gate",
"model.language_model.layers.12.mlp.shared_expert_gate",
"model.language_model.layers.12.post_attention_layernorm",
"model.language_model.layers.13.input_layernorm",
"model.language_model.layers.13.linear_attn.A_log",
"model.language_model.layers.13.linear_attn.conv1d",
"model.language_model.layers.13.linear_attn.dt",
"model.language_model.layers.13.linear_attn.in_proj_a",
"model.language_model.layers.13.linear_attn.in_proj_b",
"model.language_model.layers.13.linear_attn.in_proj_qkv",
"model.language_model.layers.13.linear_attn.in_proj_z",
"model.language_model.layers.13.linear_attn.norm",
"model.language_model.layers.13.linear_attn.out_proj",
"model.language_model.layers.13.mlp.gate",
"model.language_model.layers.13.mlp.shared_expert_gate",
"model.language_model.layers.13.post_attention_layernorm",
"model.language_model.layers.14.input_layernorm",
"model.language_model.layers.14.linear_attn.A_log",
"model.language_model.layers.14.linear_attn.conv1d",
"model.language_model.layers.14.linear_attn.dt",
"model.language_model.layers.14.linear_attn.in_proj_a",
"model.language_model.layers.14.linear_attn.in_proj_b",
"model.language_model.layers.14.linear_attn.in_proj_qkv",
"model.language_model.layers.14.linear_attn.in_proj_z",
"model.language_model.layers.14.linear_attn.norm",
"model.language_model.layers.14.linear_attn.out_proj",
"model.language_model.layers.14.mlp.gate",
"model.language_model.layers.14.mlp.shared_expert_gate",
"model.language_model.layers.14.post_attention_layernorm",
"model.language_model.layers.15.input_layernorm",
"model.language_model.layers.15.mlp.gate",
"model.language_model.layers.15.mlp.shared_expert_gate",
"model.language_model.layers.15.post_attention_layernorm",
"model.language_model.layers.15.self_attn.k_norm",
"model.language_model.layers.15.self_attn.k_proj",
"model.language_model.layers.15.self_attn.o_proj",
"model.language_model.layers.15.self_attn.q_norm",
"model.language_model.layers.15.self_attn.q_proj",
"model.language_model.layers.15.self_attn.v_proj",
"model.language_model.layers.16.input_layernorm",
"model.language_model.layers.16.linear_attn.A_log",
"model.language_model.layers.16.linear_attn.conv1d",
"model.language_model.layers.16.linear_attn.dt",
"model.language_model.layers.16.linear_attn.in_proj_a",
"model.language_model.layers.16.linear_attn.in_proj_b",
"model.language_model.layers.16.linear_attn.in_proj_qkv",
"model.language_model.layers.16.linear_attn.in_proj_z",
"model.language_model.layers.16.linear_attn.norm",
"model.language_model.layers.16.linear_attn.out_proj",
"model.language_model.layers.16.mlp.gate",
"model.language_model.layers.16.mlp.shared_expert_gate",
"model.language_model.layers.16.post_attention_layernorm",
"model.language_model.layers.17.input_layernorm",
"model.language_model.layers.17.linear_attn.A_log",
"model.language_model.layers.17.linear_attn.conv1d",
"model.language_model.layers.17.linear_attn.dt",
"model.language_model.layers.17.linear_attn.in_proj_a",
"model.language_model.layers.17.linear_attn.in_proj_b",
"model.language_model.layers.17.linear_attn.in_proj_qkv",
"model.language_model.layers.17.linear_attn.in_proj_z",
"model.language_model.layers.17.linear_attn.norm",
"model.language_model.layers.17.linear_attn.out_proj",
"model.language_model.layers.17.mlp.gate",
"model.language_model.layers.17.mlp.shared_expert_gate",
"model.language_model.layers.17.post_attention_layernorm",
"model.language_model.layers.18.input_layernorm",
"model.language_model.layers.18.linear_attn.A_log",
"model.language_model.layers.18.linear_attn.conv1d",
"model.language_model.layers.18.linear_attn.dt",
"model.language_model.layers.18.linear_attn.in_proj_a",
"model.language_model.layers.18.linear_attn.in_proj_b",
"model.language_model.layers.18.linear_attn.in_proj_qkv",
"model.language_model.layers.18.linear_attn.in_proj_z",
"model.language_model.layers.18.linear_attn.norm",
"model.language_model.layers.18.linear_attn.out_proj",
"model.language_model.layers.18.mlp.gate",
"model.language_model.layers.18.mlp.shared_expert_gate",
"model.language_model.layers.18.post_attention_layernorm",
"model.language_model.layers.19.input_layernorm",
"model.language_model.layers.19.mlp.gate",
"model.language_model.layers.19.mlp.shared_expert_gate",
"model.language_model.layers.19.post_attention_layernorm",
"model.language_model.layers.19.self_attn.k_norm",
"model.language_model.layers.19.self_attn.k_proj",
"model.language_model.layers.19.self_attn.o_proj",
"model.language_model.layers.19.self_attn.q_norm",
"model.language_model.layers.19.self_attn.q_proj",
"model.language_model.layers.19.self_attn.v_proj",
"model.language_model.layers.2.input_layernorm",
"model.language_model.layers.2.linear_attn.A_log",
"model.language_model.layers.2.linear_attn.conv1d",
"model.language_model.layers.2.linear_attn.dt",
"model.language_model.layers.2.linear_attn.in_proj_a",
"model.language_model.layers.2.linear_attn.in_proj_b",
"model.language_model.layers.2.linear_attn.in_proj_qkv",
"model.language_model.layers.2.linear_attn.in_proj_z",
"model.language_model.layers.2.linear_attn.norm",
"model.language_model.layers.2.linear_attn.out_proj",
"model.language_model.layers.2.mlp.gate",
"model.language_model.layers.2.mlp.shared_expert_gate",
"model.language_model.layers.2.post_attention_layernorm",
"model.language_model.layers.20.input_layernorm",
"model.language_model.layers.20.linear_attn.A_log",
"model.language_model.layers.20.linear_attn.conv1d",
"model.language_model.layers.20.linear_attn.dt",
"model.language_model.layers.20.linear_attn.in_proj_a",
"model.language_model.layers.20.linear_attn.in_proj_b",
"model.language_model.layers.20.linear_attn.in_proj_qkv",
"model.language_model.layers.20.linear_attn.in_proj_z",
"model.language_model.layers.20.linear_attn.norm",
"model.language_model.layers.20.linear_attn.out_proj",
"model.language_model.layers.20.mlp.gate",
"model.language_model.layers.20.mlp.shared_expert_gate",
"model.language_model.layers.20.post_attention_layernorm",
"model.language_model.layers.21.input_layernorm",
"model.language_model.layers.21.linear_attn.A_log",
"model.language_model.layers.21.linear_attn.conv1d",
"model.language_model.layers.21.linear_attn.dt",
"model.language_model.layers.21.linear_attn.in_proj_a",
"model.language_model.layers.21.linear_attn.in_proj_b",
"model.language_model.layers.21.linear_attn.in_proj_qkv",
"model.language_model.layers.21.linear_attn.in_proj_z",
"model.language_model.layers.21.linear_attn.norm",
"model.language_model.layers.21.linear_attn.out_proj",
"model.language_model.layers.21.mlp.gate",
"model.language_model.layers.21.mlp.shared_expert_gate",
"model.language_model.layers.21.post_attention_layernorm",
"model.language_model.layers.22.input_layernorm",
"model.language_model.layers.22.linear_attn.A_log",
"model.language_model.layers.22.linear_attn.conv1d",
"model.language_model.layers.22.linear_attn.dt",
"model.language_model.layers.22.linear_attn.in_proj_a",
"model.language_model.layers.22.linear_attn.in_proj_b",
"model.language_model.layers.22.linear_attn.in_proj_qkv",
"model.language_model.layers.22.linear_attn.in_proj_z",
"model.language_model.layers.22.linear_attn.norm",
"model.language_model.layers.22.linear_attn.out_proj",
"model.language_model.layers.22.mlp.gate",
"model.language_model.layers.22.mlp.shared_expert_gate",
"model.language_model.layers.22.post_attention_layernorm",
"model.language_model.layers.23.input_layernorm",
"model.language_model.layers.23.mlp.gate",
"model.language_model.layers.23.mlp.shared_expert_gate",
"model.language_model.layers.23.post_attention_layernorm",
"model.language_model.layers.23.self_attn.k_norm",
"model.language_model.layers.23.self_attn.k_proj",
"model.language_model.layers.23.self_attn.o_proj",
"model.language_model.layers.23.self_attn.q_norm",
"model.language_model.layers.23.self_attn.q_proj",
"model.language_model.layers.23.self_attn.v_proj",
"model.language_model.layers.24.input_layernorm",
"model.language_model.layers.24.linear_attn.A_log",
"model.language_model.layers.24.linear_attn.conv1d",
"model.language_model.layers.24.linear_attn.dt",
"model.language_model.layers.24.linear_attn.in_proj_a",
"model.language_model.layers.24.linear_attn.in_proj_b",
"model.language_model.layers.24.linear_attn.in_proj_qkv",
"model.language_model.layers.24.linear_attn.in_proj_z",
"model.language_model.layers.24.linear_attn.norm",
"model.language_model.layers.24.linear_attn.out_proj",
"model.language_model.layers.24.mlp.gate",
"model.language_model.layers.24.mlp.shared_expert_gate",
"model.language_model.layers.24.post_attention_layernorm",
"model.language_model.layers.25.input_layernorm",
"model.language_model.layers.25.linear_attn.A_log",
"model.language_model.layers.25.linear_attn.conv1d",
"model.language_model.layers.25.linear_attn.dt",
"model.language_model.layers.25.linear_attn.in_proj_a",
"model.language_model.layers.25.linear_attn.in_proj_b",
"model.language_model.layers.25.linear_attn.in_proj_qkv",
"model.language_model.layers.25.linear_attn.in_proj_z",
"model.language_model.layers.25.linear_attn.norm",
"model.language_model.layers.25.linear_attn.out_proj",
"model.language_model.layers.25.mlp.gate",
"model.language_model.layers.25.mlp.shared_expert_gate",
"model.language_model.layers.25.post_attention_layernorm",
"model.language_model.layers.26.input_layernorm",
"model.language_model.layers.26.linear_attn.A_log",
"model.language_model.layers.26.linear_attn.conv1d",
"model.language_model.layers.26.linear_attn.dt",
"model.language_model.layers.26.linear_attn.in_proj_a",
"model.language_model.layers.26.linear_attn.in_proj_b",
"model.language_model.layers.26.linear_attn.in_proj_qkv",
"model.language_model.layers.26.linear_attn.in_proj_z",
"model.language_model.layers.26.linear_attn.norm",
"model.language_model.layers.26.linear_attn.out_proj",
"model.language_model.layers.26.mlp.gate",
"model.language_model.layers.26.mlp.shared_expert_gate",
"model.language_model.layers.26.post_attention_layernorm",
"model.language_model.layers.27.input_layernorm",
"model.language_model.layers.27.mlp.gate",
"model.language_model.layers.27.mlp.shared_expert_gate",
"model.language_model.layers.27.post_attention_layernorm",
"model.language_model.layers.27.self_attn.k_norm",
"model.language_model.layers.27.self_attn.k_proj",
"model.language_model.layers.27.self_attn.o_proj",
"model.language_model.layers.27.self_attn.q_norm",
"model.language_model.layers.27.self_attn.q_proj",
"model.language_model.layers.27.self_attn.v_proj",
"model.language_model.layers.28.input_layernorm",
"model.language_model.layers.28.linear_attn.A_log",
"model.language_model.layers.28.linear_attn.conv1d",
"model.language_model.layers.28.linear_attn.dt",
"model.language_model.layers.28.linear_attn.in_proj_a",
"model.language_model.layers.28.linear_attn.in_proj_b",
"model.language_model.layers.28.linear_attn.in_proj_qkv",
"model.language_model.layers.28.linear_attn.in_proj_z",
"model.language_model.layers.28.linear_attn.norm",
"model.language_model.layers.28.linear_attn.out_proj",
"model.language_model.layers.28.mlp.gate",
"model.language_model.layers.28.mlp.shared_expert_gate",
"model.language_model.layers.28.post_attention_layernorm",
"model.language_model.layers.29.input_layernorm",
"model.language_model.layers.29.linear_attn.A_log",
"model.language_model.layers.29.linear_attn.conv1d",
"model.language_model.layers.29.linear_attn.dt",
"model.language_model.layers.29.linear_attn.in_proj_a",
"model.language_model.layers.29.linear_attn.in_proj_b",
"model.language_model.layers.29.linear_attn.in_proj_qkv",
"model.language_model.layers.29.linear_attn.in_proj_z",
"model.language_model.layers.29.linear_attn.norm",
"model.language_model.layers.29.linear_attn.out_proj",
"model.language_model.layers.29.mlp.gate",
"model.language_model.layers.29.mlp.shared_expert_gate",
"model.language_model.layers.29.post_attention_layernorm",
"model.language_model.layers.3.input_layernorm",
"model.language_model.layers.3.mlp.gate",
"model.language_model.layers.3.mlp.shared_expert_gate",
"model.language_model.layers.3.post_attention_layernorm",
"model.language_model.layers.3.self_attn.k_norm",
"model.language_model.layers.3.self_attn.k_proj",
"model.language_model.layers.3.self_attn.o_proj",
"model.language_model.layers.3.self_attn.q_norm",
"model.language_model.layers.3.self_attn.q_proj",
"model.language_model.layers.3.self_attn.v_proj",
"model.language_model.layers.30.input_layernorm",
"model.language_model.layers.30.linear_attn.A_log",
"model.language_model.layers.30.linear_attn.conv1d",
"model.language_model.layers.30.linear_attn.dt",
"model.language_model.layers.30.linear_attn.in_proj_a",
"model.language_model.layers.30.linear_attn.in_proj_b",
"model.language_model.layers.30.linear_attn.in_proj_qkv",
"model.language_model.layers.30.linear_attn.in_proj_z",
"model.language_model.layers.30.linear_attn.norm",
"model.language_model.layers.30.linear_attn.out_proj",
"model.language_model.layers.30.mlp.gate",
"model.language_model.layers.30.mlp.shared_expert_gate",
"model.language_model.layers.30.post_attention_layernorm",
"model.language_model.layers.31.input_layernorm",
"model.language_model.layers.31.mlp.gate",
"model.language_model.layers.31.mlp.shared_expert_gate",
"model.language_model.layers.31.post_attention_layernorm",
"model.language_model.layers.31.self_attn.k_norm",
"model.language_model.layers.31.self_attn.k_proj",
"model.language_model.layers.31.self_attn.o_proj",
"model.language_model.layers.31.self_attn.q_norm",
"model.language_model.layers.31.self_attn.q_proj",
"model.language_model.layers.31.self_attn.v_proj",
"model.language_model.layers.32.input_layernorm",
"model.language_model.layers.32.linear_attn.A_log",
"model.language_model.layers.32.linear_attn.conv1d",
"model.language_model.layers.32.linear_attn.dt",
"model.language_model.layers.32.linear_attn.in_proj_a",
"model.language_model.layers.32.linear_attn.in_proj_b",
"model.language_model.layers.32.linear_attn.in_proj_qkv",
"model.language_model.layers.32.linear_attn.in_proj_z",
"model.language_model.layers.32.linear_attn.norm",
"model.language_model.layers.32.linear_attn.out_proj",
"model.language_model.layers.32.mlp.gate",
"model.language_model.layers.32.mlp.shared_expert_gate",
"model.language_model.layers.32.post_attention_layernorm",
"model.language_model.layers.33.input_layernorm",
"model.language_model.layers.33.linear_attn.A_log",
"model.language_model.layers.33.linear_attn.conv1d",
"model.language_model.layers.33.linear_attn.dt",
"model.language_model.layers.33.linear_attn.in_proj_a",
"model.language_model.layers.33.linear_attn.in_proj_b",
"model.language_model.layers.33.linear_attn.in_proj_qkv",
"model.language_model.layers.33.linear_attn.in_proj_z",
"model.language_model.layers.33.linear_attn.norm",
"model.language_model.layers.33.linear_attn.out_proj",
"model.language_model.layers.33.mlp.gate",
"model.language_model.layers.33.mlp.shared_expert_gate",
"model.language_model.layers.33.post_attention_layernorm",
"model.language_model.layers.34.input_layernorm",
"model.language_model.layers.34.linear_attn.A_log",
"model.language_model.layers.34.linear_attn.conv1d",
"model.language_model.layers.34.linear_attn.dt",
"model.language_model.layers.34.linear_attn.in_proj_a",
"model.language_model.layers.34.linear_attn.in_proj_b",
"model.language_model.layers.34.linear_attn.in_proj_qkv",
"model.language_model.layers.34.linear_attn.in_proj_z",
"model.language_model.layers.34.linear_attn.norm",
"model.language_model.layers.34.linear_attn.out_proj",
"model.language_model.layers.34.mlp.gate",
"model.language_model.layers.34.mlp.shared_expert_gate",
"model.language_model.layers.34.post_attention_layernorm",
"model.language_model.layers.35.input_layernorm",
"model.language_model.layers.35.mlp.gate",
"model.language_model.layers.35.mlp.shared_expert_gate",
"model.language_model.layers.35.post_attention_layernorm",
"model.language_model.layers.35.self_attn.k_norm",
"model.language_model.layers.35.self_attn.k_proj",
"model.language_model.layers.35.self_attn.o_proj",
"model.language_model.layers.35.self_attn.q_norm",
"model.language_model.layers.35.self_attn.q_proj",
"model.language_model.layers.35.self_attn.v_proj",
"model.language_model.layers.36.input_layernorm",
"model.language_model.layers.36.linear_attn.A_log",
"model.language_model.layers.36.linear_attn.conv1d",
"model.language_model.layers.36.linear_attn.dt",
"model.language_model.layers.36.linear_attn.in_proj_a",
"model.language_model.layers.36.linear_attn.in_proj_b",
"model.language_model.layers.36.linear_attn.in_proj_qkv",
"model.language_model.layers.36.linear_attn.in_proj_z",
"model.language_model.layers.36.linear_attn.norm",
"model.language_model.layers.36.linear_attn.out_proj",
"model.language_model.layers.36.mlp.gate",
"model.language_model.layers.36.mlp.shared_expert_gate",
"model.language_model.layers.36.post_attention_layernorm",
"model.language_model.layers.37.input_layernorm",
"model.language_model.layers.37.linear_attn.A_log",
"model.language_model.layers.37.linear_attn.conv1d",
"model.language_model.layers.37.linear_attn.dt",
"model.language_model.layers.37.linear_attn.in_proj_a",
"model.language_model.layers.37.linear_attn.in_proj_b",
"model.language_model.layers.37.linear_attn.in_proj_qkv",
"model.language_model.layers.37.linear_attn.in_proj_z",
"model.language_model.layers.37.linear_attn.norm",
"model.language_model.layers.37.linear_attn.out_proj",
"model.language_model.layers.37.mlp.gate",
"model.language_model.layers.37.mlp.shared_expert_gate",
"model.language_model.layers.37.post_attention_layernorm",
"model.language_model.layers.38.input_layernorm",
"model.language_model.layers.38.linear_attn.A_log",
"model.language_model.layers.38.linear_attn.conv1d",
"model.language_model.layers.38.linear_attn.dt",
"model.language_model.layers.38.linear_attn.in_proj_a",
"model.language_model.layers.38.linear_attn.in_proj_b",
"model.language_model.layers.38.linear_attn.in_proj_qkv",
"model.language_model.layers.38.linear_attn.in_proj_z",
"model.language_model.layers.38.linear_attn.norm",
"model.language_model.layers.38.linear_attn.out_proj",
"model.language_model.layers.38.mlp.gate",
"model.language_model.layers.38.mlp.shared_expert_gate",
"model.language_model.layers.38.post_attention_layernorm",
"model.language_model.layers.39.input_layernorm",
"model.language_model.layers.39.mlp.gate",
"model.language_model.layers.39.mlp.shared_expert_gate",
"model.language_model.layers.39.post_attention_layernorm",
"model.language_model.layers.39.self_attn.k_norm",
"model.language_model.layers.39.self_attn.k_proj",
"model.language_model.layers.39.self_attn.o_proj",
"model.language_model.layers.39.self_attn.q_norm",
"model.language_model.layers.39.self_attn.q_proj",
"model.language_model.layers.39.self_attn.v_proj",
"model.language_model.layers.4.input_layernorm",
"model.language_model.layers.4.linear_attn.A_log",
"model.language_model.layers.4.linear_attn.conv1d",
"model.language_model.layers.4.linear_attn.dt",
"model.language_model.layers.4.linear_attn.in_proj_a",
"model.language_model.layers.4.linear_attn.in_proj_b",
"model.language_model.layers.4.linear_attn.in_proj_qkv",
"model.language_model.layers.4.linear_attn.in_proj_z",
"model.language_model.layers.4.linear_attn.norm",
"model.language_model.layers.4.linear_attn.out_proj",
"model.language_model.layers.4.mlp.gate",
"model.language_model.layers.4.mlp.shared_expert_gate",
"model.language_model.layers.4.post_attention_layernorm",
"model.language_model.layers.40.input_layernorm",
"model.language_model.layers.40.linear_attn.A_log",
"model.language_model.layers.40.linear_attn.conv1d",
"model.language_model.layers.40.linear_attn.dt",
"model.language_model.layers.40.linear_attn.in_proj_a",
"model.language_model.layers.40.linear_attn.in_proj_b",
"model.language_model.layers.40.linear_attn.in_proj_qkv",
"model.language_model.layers.40.linear_attn.in_proj_z",
"model.language_model.layers.40.linear_attn.norm",
"model.language_model.layers.40.linear_attn.out_proj",
"model.language_model.layers.40.mlp.gate",
"model.language_model.layers.40.mlp.shared_expert_gate",
"model.language_model.layers.40.post_attention_layernorm",
"model.language_model.layers.41.input_layernorm",
"model.language_model.layers.41.linear_attn.A_log",
"model.language_model.layers.41.linear_attn.conv1d",
"model.language_model.layers.41.linear_attn.dt",
"model.language_model.layers.41.linear_attn.in_proj_a",
"model.language_model.layers.41.linear_attn.in_proj_b",
"model.language_model.layers.41.linear_attn.in_proj_qkv",
"model.language_model.layers.41.linear_attn.in_proj_z",
"model.language_model.layers.41.linear_attn.norm",
"model.language_model.layers.41.linear_attn.out_proj",
"model.language_model.layers.41.mlp.gate",
"model.language_model.layers.41.mlp.shared_expert_gate",
"model.language_model.layers.41.post_attention_layernorm",
"model.language_model.layers.42.input_layernorm",
"model.language_model.layers.42.linear_attn.A_log",
"model.language_model.layers.42.linear_attn.conv1d",
"model.language_model.layers.42.linear_attn.dt",
"model.language_model.layers.42.linear_attn.in_proj_a",
"model.language_model.layers.42.linear_attn.in_proj_b",
"model.language_model.layers.42.linear_attn.in_proj_qkv",
"model.language_model.layers.42.linear_attn.in_proj_z",
"model.language_model.layers.42.linear_attn.norm",
"model.language_model.layers.42.linear_attn.out_proj",
"model.language_model.layers.42.mlp.gate",
"model.language_model.layers.42.mlp.shared_expert_gate",
"model.language_model.layers.42.post_attention_layernorm",
"model.language_model.layers.43.input_layernorm",
"model.language_model.layers.43.mlp.gate",
"model.language_model.layers.43.mlp.shared_expert_gate",
"model.language_model.layers.43.post_attention_layernorm",
"model.language_model.layers.43.self_attn.k_norm",
"model.language_model.layers.43.self_attn.k_proj",
"model.language_model.layers.43.self_attn.o_proj",
"model.language_model.layers.43.self_attn.q_norm",
"model.language_model.layers.43.self_attn.q_proj",
"model.language_model.layers.43.self_attn.v_proj",
"model.language_model.layers.44.input_layernorm",
"model.language_model.layers.44.linear_attn.A_log",
"model.language_model.layers.44.linear_attn.conv1d",
"model.language_model.layers.44.linear_attn.dt",
"model.language_model.layers.44.linear_attn.in_proj_a",
"model.language_model.layers.44.linear_attn.in_proj_b",
"model.language_model.layers.44.linear_attn.in_proj_qkv",
"model.language_model.layers.44.linear_attn.in_proj_z",
"model.language_model.layers.44.linear_attn.norm",
"model.language_model.layers.44.linear_attn.out_proj",
"model.language_model.layers.44.mlp.gate",
"model.language_model.layers.44.mlp.shared_expert_gate",
"model.language_model.layers.44.post_attention_layernorm",
"model.language_model.layers.45.input_layernorm",
"model.language_model.layers.45.linear_attn.A_log",
"model.language_model.layers.45.linear_attn.conv1d",
"model.language_model.layers.45.linear_attn.dt",
"model.language_model.layers.45.linear_attn.in_proj_a",
"model.language_model.layers.45.linear_attn.in_proj_b",
"model.language_model.layers.45.linear_attn.in_proj_qkv",
"model.language_model.layers.45.linear_attn.in_proj_z",
"model.language_model.layers.45.linear_attn.norm",
"model.language_model.layers.45.linear_attn.out_proj",
"model.language_model.layers.45.mlp.gate",
"model.language_model.layers.45.mlp.shared_expert_gate",
"model.language_model.layers.45.post_attention_layernorm",
"model.language_model.layers.46.input_layernorm",
"model.language_model.layers.46.linear_attn.A_log",
"model.language_model.layers.46.linear_attn.conv1d",
"model.language_model.layers.46.linear_attn.dt",
"model.language_model.layers.46.linear_attn.in_proj_a",
"model.language_model.layers.46.linear_attn.in_proj_b",
"model.language_model.layers.46.linear_attn.in_proj_qkv",
"model.language_model.layers.46.linear_attn.in_proj_z",
"model.language_model.layers.46.linear_attn.norm",
"model.language_model.layers.46.linear_attn.out_proj",
"model.language_model.layers.46.mlp.gate",
"model.language_model.layers.46.mlp.shared_expert_gate",
"model.language_model.layers.46.post_attention_layernorm",
"model.language_model.layers.47.input_layernorm",
"model.language_model.layers.47.mlp.gate",
"model.language_model.layers.47.mlp.shared_expert_gate",
"model.language_model.layers.47.post_attention_layernorm",
"model.language_model.layers.47.self_attn.k_norm",
"model.language_model.layers.47.self_attn.k_proj",
"model.language_model.layers.47.self_attn.o_proj",
"model.language_model.layers.47.self_attn.q_norm",
"model.language_model.layers.47.self_attn.q_proj",
"model.language_model.layers.47.self_attn.v_proj",
"model.language_model.layers.48.input_layernorm",
"model.language_model.layers.48.linear_attn.A_log",
"model.language_model.layers.48.linear_attn.conv1d",
"model.language_model.layers.48.linear_attn.dt",
"model.language_model.layers.48.linear_attn.in_proj_a",
"model.language_model.layers.48.linear_attn.in_proj_b",
"model.language_model.layers.48.linear_attn.in_proj_qkv",
"model.language_model.layers.48.linear_attn.in_proj_z",
"model.language_model.layers.48.linear_attn.norm",
"model.language_model.layers.48.linear_attn.out_proj",
"model.language_model.layers.48.mlp.gate",
"model.language_model.layers.48.mlp.shared_expert_gate",
"model.language_model.layers.48.post_attention_layernorm",
"model.language_model.layers.49.input_layernorm",
"model.language_model.layers.49.linear_attn.A_log",
"model.language_model.layers.49.linear_attn.conv1d",
"model.language_model.layers.49.linear_attn.dt",
"model.language_model.layers.49.linear_attn.in_proj_a",
"model.language_model.layers.49.linear_attn.in_proj_b",
"model.language_model.layers.49.linear_attn.in_proj_qkv",
"model.language_model.layers.49.linear_attn.in_proj_z",
"model.language_model.layers.49.linear_attn.norm",
"model.language_model.layers.49.linear_attn.out_proj",
"model.language_model.layers.49.mlp.gate",
"model.language_model.layers.49.mlp.shared_expert_gate",
"model.language_model.layers.49.post_attention_layernorm",
"model.language_model.layers.5.input_layernorm",
"model.language_model.layers.5.linear_attn.A_log",
"model.language_model.layers.5.linear_attn.conv1d",
"model.language_model.layers.5.linear_attn.dt",
"model.language_model.layers.5.linear_attn.in_proj_a",
"model.language_model.layers.5.linear_attn.in_proj_b",
"model.language_model.layers.5.linear_attn.in_proj_qkv",
"model.language_model.layers.5.linear_attn.in_proj_z",
"model.language_model.layers.5.linear_attn.norm",
"model.language_model.layers.5.linear_attn.out_proj",
"model.language_model.layers.5.mlp.gate",
"model.language_model.layers.5.mlp.shared_expert_gate",
"model.language_model.layers.5.post_attention_layernorm",
"model.language_model.layers.50.input_layernorm",
"model.language_model.layers.50.linear_attn.A_log",
"model.language_model.layers.50.linear_attn.conv1d",
"model.language_model.layers.50.linear_attn.dt",
"model.language_model.layers.50.linear_attn.in_proj_a",
"model.language_model.layers.50.linear_attn.in_proj_b",
"model.language_model.layers.50.linear_attn.in_proj_qkv",
"model.language_model.layers.50.linear_attn.in_proj_z",
"model.language_model.layers.50.linear_attn.norm",
"model.language_model.layers.50.linear_attn.out_proj",
"model.language_model.layers.50.mlp.gate",
"model.language_model.layers.50.mlp.shared_expert_gate",
"model.language_model.layers.50.post_attention_layernorm",
"model.language_model.layers.51.input_layernorm",
"model.language_model.layers.51.mlp.gate",
"model.language_model.layers.51.mlp.shared_expert_gate",
"model.language_model.layers.51.post_attention_layernorm",
"model.language_model.layers.51.self_attn.k_norm",
"model.language_model.layers.51.self_attn.k_proj",
"model.language_model.layers.51.self_attn.o_proj",
"model.language_model.layers.51.self_attn.q_norm",
"model.language_model.layers.51.self_attn.q_proj",
"model.language_model.layers.51.self_attn.v_proj",
"model.language_model.layers.52.input_layernorm",
"model.language_model.layers.52.linear_attn.A_log",
"model.language_model.layers.52.linear_attn.conv1d",
"model.language_model.layers.52.linear_attn.dt",
"model.language_model.layers.52.linear_attn.in_proj_a",
"model.language_model.layers.52.linear_attn.in_proj_b",
"model.language_model.layers.52.linear_attn.in_proj_qkv",
"model.language_model.layers.52.linear_attn.in_proj_z",
"model.language_model.layers.52.linear_attn.norm",
"model.language_model.layers.52.linear_attn.out_proj",
"model.language_model.layers.52.mlp.gate",
"model.language_model.layers.52.mlp.shared_expert_gate",
"model.language_model.layers.52.post_attention_layernorm",
"model.language_model.layers.53.input_layernorm",
"model.language_model.layers.53.linear_attn.A_log",
"model.language_model.layers.53.linear_attn.conv1d",
"model.language_model.layers.53.linear_attn.dt",
"model.language_model.layers.53.linear_attn.in_proj_a",
"model.language_model.layers.53.linear_attn.in_proj_b",
"model.language_model.layers.53.linear_attn.in_proj_qkv",
"model.language_model.layers.53.linear_attn.in_proj_z",
"model.language_model.layers.53.linear_attn.norm",
"model.language_model.layers.53.linear_attn.out_proj",
"model.language_model.layers.53.mlp.gate",
"model.language_model.layers.53.mlp.shared_expert_gate",
"model.language_model.layers.53.post_attention_layernorm",
"model.language_model.layers.54.input_layernorm",
"model.language_model.layers.54.linear_attn.A_log",
"model.language_model.layers.54.linear_attn.conv1d",
"model.language_model.layers.54.linear_attn.dt",
"model.language_model.layers.54.linear_attn.in_proj_a",
"model.language_model.layers.54.linear_attn.in_proj_b",
"model.language_model.layers.54.linear_attn.in_proj_qkv",
"model.language_model.layers.54.linear_attn.in_proj_z",
"model.language_model.layers.54.linear_attn.norm",
"model.language_model.layers.54.linear_attn.out_proj",
"model.language_model.layers.54.mlp.gate",
"model.language_model.layers.54.mlp.shared_expert_gate",
"model.language_model.layers.54.post_attention_layernorm",
"model.language_model.layers.55.input_layernorm",
"model.language_model.layers.55.mlp.gate",
"model.language_model.layers.55.mlp.shared_expert_gate",
"model.language_model.layers.55.post_attention_layernorm",
"model.language_model.layers.55.self_attn.k_norm",
"model.language_model.layers.55.self_attn.k_proj",
"model.language_model.layers.55.self_attn.o_proj",
"model.language_model.layers.55.self_attn.q_norm",
"model.language_model.layers.55.self_attn.q_proj",
"model.language_model.layers.55.self_attn.v_proj",
"model.language_model.layers.56.input_layernorm",
"model.language_model.layers.56.linear_attn.A_log",
"model.language_model.layers.56.linear_attn.conv1d",
"model.language_model.layers.56.linear_attn.dt",
"model.language_model.layers.56.linear_attn.in_proj_a",
"model.language_model.layers.56.linear_attn.in_proj_b",
"model.language_model.layers.56.linear_attn.in_proj_qkv",
"model.language_model.layers.56.linear_attn.in_proj_z",
"model.language_model.layers.56.linear_attn.norm",
"model.language_model.layers.56.linear_attn.out_proj",
"model.language_model.layers.56.mlp.gate",
"model.language_model.layers.56.mlp.shared_expert_gate",
"model.language_model.layers.56.post_attention_layernorm",
"model.language_model.layers.57.input_layernorm",
"model.language_model.layers.57.linear_attn.A_log",
"model.language_model.layers.57.linear_attn.conv1d",
"model.language_model.layers.57.linear_attn.dt",
"model.language_model.layers.57.linear_attn.in_proj_a",
"model.language_model.layers.57.linear_attn.in_proj_b",
"model.language_model.layers.57.linear_attn.in_proj_qkv",
"model.language_model.layers.57.linear_attn.in_proj_z",
"model.language_model.layers.57.linear_attn.norm",
"model.language_model.layers.57.linear_attn.out_proj",
"model.language_model.layers.57.mlp.gate",
"model.language_model.layers.57.mlp.shared_expert_gate",
"model.language_model.layers.57.post_attention_layernorm",
"model.language_model.layers.58.input_layernorm",
"model.language_model.layers.58.linear_attn.A_log",
"model.language_model.layers.58.linear_attn.conv1d",
"model.language_model.layers.58.linear_attn.dt",
"model.language_model.layers.58.linear_attn.in_proj_a",
"model.language_model.layers.58.linear_attn.in_proj_b",
"model.language_model.layers.58.linear_attn.in_proj_qkv",
"model.language_model.layers.58.linear_attn.in_proj_z",
"model.language_model.layers.58.linear_attn.norm",
"model.language_model.layers.58.linear_attn.out_proj",
"model.language_model.layers.58.mlp.gate",
"model.language_model.layers.58.mlp.shared_expert_gate",
"model.language_model.layers.58.post_attention_layernorm",
"model.language_model.layers.59.input_layernorm",
"model.language_model.layers.59.mlp.gate",
"model.language_model.layers.59.mlp.shared_expert_gate",
"model.language_model.layers.59.post_attention_layernorm",
"model.language_model.layers.59.self_attn.k_norm",
"model.language_model.layers.59.self_attn.k_proj",
"model.language_model.layers.59.self_attn.o_proj",
"model.language_model.layers.59.self_attn.q_norm",
"model.language_model.layers.59.self_attn.q_proj",
"model.language_model.layers.59.self_attn.v_proj",
"model.language_model.layers.6.input_layernorm",
"model.language_model.layers.6.linear_attn.A_log",
"model.language_model.layers.6.linear_attn.conv1d",
"model.language_model.layers.6.linear_attn.dt",
"model.language_model.layers.6.linear_attn.in_proj_a",
"model.language_model.layers.6.linear_attn.in_proj_b",
"model.language_model.layers.6.linear_attn.in_proj_qkv",
"model.language_model.layers.6.linear_attn.in_proj_z",
"model.language_model.layers.6.linear_attn.norm",
"model.language_model.layers.6.linear_attn.out_proj",
"model.language_model.layers.6.mlp.gate",
"model.language_model.layers.6.mlp.shared_expert_gate",
"model.language_model.layers.6.post_attention_layernorm",
"model.language_model.layers.7.input_layernorm",
"model.language_model.layers.7.mlp.gate",
"model.language_model.layers.7.mlp.shared_expert_gate",
"model.language_model.layers.7.post_attention_layernorm",
"model.language_model.layers.7.self_attn.k_norm",
"model.language_model.layers.7.self_attn.k_proj",
"model.language_model.layers.7.self_attn.o_proj",
"model.language_model.layers.7.self_attn.q_norm",
"model.language_model.layers.7.self_attn.q_proj",
"model.language_model.layers.7.self_attn.v_proj",
"model.language_model.layers.8.input_layernorm",
"model.language_model.layers.8.linear_attn.A_log",
"model.language_model.layers.8.linear_attn.conv1d",
"model.language_model.layers.8.linear_attn.dt",
"model.language_model.layers.8.linear_attn.in_proj_a",
"model.language_model.layers.8.linear_attn.in_proj_b",
"model.language_model.layers.8.linear_attn.in_proj_qkv",
"model.language_model.layers.8.linear_attn.in_proj_z",
"model.language_model.layers.8.linear_attn.norm",
"model.language_model.layers.8.linear_attn.out_proj",
"model.language_model.layers.8.mlp.gate",
"model.language_model.layers.8.mlp.shared_expert_gate",
"model.language_model.layers.8.post_attention_layernorm",
"model.language_model.layers.9.input_layernorm",
"model.language_model.layers.9.linear_attn.A_log",
"model.language_model.layers.9.linear_attn.conv1d",
"model.language_model.layers.9.linear_attn.dt",
"model.language_model.layers.9.linear_attn.in_proj_a",
"model.language_model.layers.9.linear_attn.in_proj_b",
"model.language_model.layers.9.linear_attn.in_proj_qkv",
"model.language_model.layers.9.linear_attn.in_proj_z",
"model.language_model.layers.9.linear_attn.norm",
"model.language_model.layers.9.linear_attn.out_proj",
"model.language_model.layers.9.mlp.gate",
"model.language_model.layers.9.mlp.shared_expert_gate",
"model.language_model.layers.9.post_attention_layernorm",
"model.language_model.norm",
"model.visual.blocks.0.attn.proj",
"model.visual.blocks.0.attn.qkv",
"model.visual.blocks.0.mlp.linear_fc1",
"model.visual.blocks.0.mlp.linear_fc2",
"model.visual.blocks.0.norm1",
"model.visual.blocks.0.norm2",
"model.visual.blocks.1.attn.proj",
"model.visual.blocks.1.attn.qkv",
"model.visual.blocks.1.mlp.linear_fc1",
"model.visual.blocks.1.mlp.linear_fc2",
"model.visual.blocks.1.norm1",
"model.visual.blocks.1.norm2",
"model.visual.blocks.10.attn.proj",
"model.visual.blocks.10.attn.qkv",
"model.visual.blocks.10.mlp.linear_fc1",
"model.visual.blocks.10.mlp.linear_fc2",
"model.visual.blocks.10.norm1",
"model.visual.blocks.10.norm2",
"model.visual.blocks.11.attn.proj",
"model.visual.blocks.11.attn.qkv",
"model.visual.blocks.11.mlp.linear_fc1",
"model.visual.blocks.11.mlp.linear_fc2",
"model.visual.blocks.11.norm1",
"model.visual.blocks.11.norm2",
"model.visual.blocks.12.attn.proj",
"model.visual.blocks.12.attn.qkv",
"model.visual.blocks.12.mlp.linear_fc1",
"model.visual.blocks.12.mlp.linear_fc2",
"model.visual.blocks.12.norm1",
"model.visual.blocks.12.norm2",
"model.visual.blocks.13.attn.proj",
"model.visual.blocks.13.attn.qkv",
"model.visual.blocks.13.mlp.linear_fc1",
"model.visual.blocks.13.mlp.linear_fc2",
"model.visual.blocks.13.norm1",
"model.visual.blocks.13.norm2",
"model.visual.blocks.14.attn.proj",
"model.visual.blocks.14.attn.qkv",
"model.visual.blocks.14.mlp.linear_fc1",
"model.visual.blocks.14.mlp.linear_fc2",
"model.visual.blocks.14.norm1",
"model.visual.blocks.14.norm2",
"model.visual.blocks.15.attn.proj",
"model.visual.blocks.15.attn.qkv",
"model.visual.blocks.15.mlp.linear_fc1",
"model.visual.blocks.15.mlp.linear_fc2",
"model.visual.blocks.15.norm1",
"model.visual.blocks.15.norm2",
"model.visual.blocks.16.attn.proj",
"model.visual.blocks.16.attn.qkv",
"model.visual.blocks.16.mlp.linear_fc1",
"model.visual.blocks.16.mlp.linear_fc2",
"model.visual.blocks.16.norm1",
"model.visual.blocks.16.norm2",
"model.visual.blocks.17.attn.proj",
"model.visual.blocks.17.attn.qkv",
"model.visual.blocks.17.mlp.linear_fc1",
"model.visual.blocks.17.mlp.linear_fc2",
"model.visual.blocks.17.norm1",
"model.visual.blocks.17.norm2",
"model.visual.blocks.18.attn.proj",
"model.visual.blocks.18.attn.qkv",
"model.visual.blocks.18.mlp.linear_fc1",
"model.visual.blocks.18.mlp.linear_fc2",
"model.visual.blocks.18.norm1",
"model.visual.blocks.18.norm2",
"model.visual.blocks.19.attn.proj",
"model.visual.blocks.19.attn.qkv",
"model.visual.blocks.19.mlp.linear_fc1",
"model.visual.blocks.19.mlp.linear_fc2",
"model.visual.blocks.19.norm1",
"model.visual.blocks.19.norm2",
"model.visual.blocks.2.attn.proj",
"model.visual.blocks.2.attn.qkv",
"model.visual.blocks.2.mlp.linear_fc1",
"model.visual.blocks.2.mlp.linear_fc2",
"model.visual.blocks.2.norm1",
"model.visual.blocks.2.norm2",
"model.visual.blocks.20.attn.proj",
"model.visual.blocks.20.attn.qkv",
"model.visual.blocks.20.mlp.linear_fc1",
"model.visual.blocks.20.mlp.linear_fc2",
"model.visual.blocks.20.norm1",
"model.visual.blocks.20.norm2",
"model.visual.blocks.21.attn.proj",
"model.visual.blocks.21.attn.qkv",
"model.visual.blocks.21.mlp.linear_fc1",
"model.visual.blocks.21.mlp.linear_fc2",
"model.visual.blocks.21.norm1",
"model.visual.blocks.21.norm2",
"model.visual.blocks.22.attn.proj",
"model.visual.blocks.22.attn.qkv",
"model.visual.blocks.22.mlp.linear_fc1",
"model.visual.blocks.22.mlp.linear_fc2",
"model.visual.blocks.22.norm1",
"model.visual.blocks.22.norm2",
"model.visual.blocks.23.attn.proj",
"model.visual.blocks.23.attn.qkv",
"model.visual.blocks.23.mlp.linear_fc1",
"model.visual.blocks.23.mlp.linear_fc2",
"model.visual.blocks.23.norm1",
"model.visual.blocks.23.norm2",
"model.visual.blocks.24.attn.proj",
"model.visual.blocks.24.attn.qkv",
"model.visual.blocks.24.mlp.linear_fc1",
"model.visual.blocks.24.mlp.linear_fc2",
"model.visual.blocks.24.norm1",
"model.visual.blocks.24.norm2",
"model.visual.blocks.25.attn.proj",
"model.visual.blocks.25.attn.qkv",
"model.visual.blocks.25.mlp.linear_fc1",
"model.visual.blocks.25.mlp.linear_fc2",
"model.visual.blocks.25.norm1",
"model.visual.blocks.25.norm2",
"model.visual.blocks.26.attn.proj",
"model.visual.blocks.26.attn.qkv",
"model.visual.blocks.26.mlp.linear_fc1",
"model.visual.blocks.26.mlp.linear_fc2",
"model.visual.blocks.26.norm1",
"model.visual.blocks.26.norm2",
"model.visual.blocks.3.attn.proj",
"model.visual.blocks.3.attn.qkv",
"model.visual.blocks.3.mlp.linear_fc1",
"model.visual.blocks.3.mlp.linear_fc2",
"model.visual.blocks.3.norm1",
"model.visual.blocks.3.norm2",
"model.visual.blocks.4.attn.proj",
"model.visual.blocks.4.attn.qkv",
"model.visual.blocks.4.mlp.linear_fc1",
"model.visual.blocks.4.mlp.linear_fc2",
"model.visual.blocks.4.norm1",
"model.visual.blocks.4.norm2",
"model.visual.blocks.5.attn.proj",
"model.visual.blocks.5.attn.qkv",
"model.visual.blocks.5.mlp.linear_fc1",
"model.visual.blocks.5.mlp.linear_fc2",
"model.visual.blocks.5.norm1",
"model.visual.blocks.5.norm2",
"model.visual.blocks.6.attn.proj",
"model.visual.blocks.6.attn.qkv",
"model.visual.blocks.6.mlp.linear_fc1",
"model.visual.blocks.6.mlp.linear_fc2",
"model.visual.blocks.6.norm1",
"model.visual.blocks.6.norm2",
"model.visual.blocks.7.attn.proj",
"model.visual.blocks.7.attn.qkv",
"model.visual.blocks.7.mlp.linear_fc1",
"model.visual.blocks.7.mlp.linear_fc2",
"model.visual.blocks.7.norm1",
"model.visual.blocks.7.norm2",
"model.visual.blocks.8.attn.proj",
"model.visual.blocks.8.attn.qkv",
"model.visual.blocks.8.mlp.linear_fc1",
"model.visual.blocks.8.mlp.linear_fc2",
"model.visual.blocks.8.norm1",
"model.visual.blocks.8.norm2",
"model.visual.blocks.9.attn.proj",
"model.visual.blocks.9.attn.qkv",
"model.visual.blocks.9.mlp.linear_fc1",
"model.visual.blocks.9.mlp.linear_fc2",
"model.visual.blocks.9.norm1",
"model.visual.blocks.9.norm2",
"model.visual.merger.linear_fc1",
"model.visual.merger.linear_fc2",
"model.visual.merger.norm",
"model.visual.patch_embed.proj",
"model.visual.pos_embed",
"mtp.fc",
"mtp.layers.0.input_layernorm",
"mtp.layers.0.mlp.experts.0.down_proj",
"mtp.layers.0.mlp.experts.0.gate_proj",
"mtp.layers.0.mlp.experts.0.up_proj",
"mtp.layers.0.mlp.experts.1.down_proj",
"mtp.layers.0.mlp.experts.1.gate_proj",
"mtp.layers.0.mlp.experts.1.up_proj",
"mtp.layers.0.mlp.experts.10.down_proj",
"mtp.layers.0.mlp.experts.10.gate_proj",
"mtp.layers.0.mlp.experts.10.up_proj",
"mtp.layers.0.mlp.experts.100.down_proj",
"mtp.layers.0.mlp.experts.100.gate_proj",
"mtp.layers.0.mlp.experts.100.up_proj",
"mtp.layers.0.mlp.experts.101.down_proj",
"mtp.layers.0.mlp.experts.101.gate_proj",
"mtp.layers.0.mlp.experts.101.up_proj",
"mtp.layers.0.mlp.experts.102.down_proj",
"mtp.layers.0.mlp.experts.102.gate_proj",
"mtp.layers.0.mlp.experts.102.up_proj",
"mtp.layers.0.mlp.experts.103.down_proj",
"mtp.layers.0.mlp.experts.103.gate_proj",
"mtp.layers.0.mlp.experts.103.up_proj",
"mtp.layers.0.mlp.experts.104.down_proj",
"mtp.layers.0.mlp.experts.104.gate_proj",
"mtp.layers.0.mlp.experts.104.up_proj",
"mtp.layers.0.mlp.experts.105.down_proj",
"mtp.layers.0.mlp.experts.105.gate_proj",
"mtp.layers.0.mlp.experts.105.up_proj",
"mtp.layers.0.mlp.experts.106.down_proj",
"mtp.layers.0.mlp.experts.106.gate_proj",
"mtp.layers.0.mlp.experts.106.up_proj",
"mtp.layers.0.mlp.experts.107.down_proj",
"mtp.layers.0.mlp.experts.107.gate_proj",
"mtp.layers.0.mlp.experts.107.up_proj",
"mtp.layers.0.mlp.experts.108.down_proj",
"mtp.layers.0.mlp.experts.108.gate_proj",
"mtp.layers.0.mlp.experts.108.up_proj",
"mtp.layers.0.mlp.experts.109.down_proj",
"mtp.layers.0.mlp.experts.109.gate_proj",
"mtp.layers.0.mlp.experts.109.up_proj",
"mtp.layers.0.mlp.experts.11.down_proj",
"mtp.layers.0.mlp.experts.11.gate_proj",
"mtp.layers.0.mlp.experts.11.up_proj",
"mtp.layers.0.mlp.experts.110.down_proj",
"mtp.layers.0.mlp.experts.110.gate_proj",
"mtp.layers.0.mlp.experts.110.up_proj",
"mtp.layers.0.mlp.experts.111.down_proj",
"mtp.layers.0.mlp.experts.111.gate_proj",
"mtp.layers.0.mlp.experts.111.up_proj",
"mtp.layers.0.mlp.experts.112.down_proj",
"mtp.layers.0.mlp.experts.112.gate_proj",
"mtp.layers.0.mlp.experts.112.up_proj",
"mtp.layers.0.mlp.experts.113.down_proj",
"mtp.layers.0.mlp.experts.113.gate_proj",
"mtp.layers.0.mlp.experts.113.up_proj",
"mtp.layers.0.mlp.experts.114.down_proj",
"mtp.layers.0.mlp.experts.114.gate_proj",
"mtp.layers.0.mlp.experts.114.up_proj",
"mtp.layers.0.mlp.experts.115.down_proj",
"mtp.layers.0.mlp.experts.115.gate_proj",
"mtp.layers.0.mlp.experts.115.up_proj",
"mtp.layers.0.mlp.experts.116.down_proj",
"mtp.layers.0.mlp.experts.116.gate_proj",
"mtp.layers.0.mlp.experts.116.up_proj",
"mtp.layers.0.mlp.experts.117.down_proj",
"mtp.layers.0.mlp.experts.117.gate_proj",
"mtp.layers.0.mlp.experts.117.up_proj",
"mtp.layers.0.mlp.experts.118.down_proj",
"mtp.layers.0.mlp.experts.118.gate_proj",
"mtp.layers.0.mlp.experts.118.up_proj",
"mtp.layers.0.mlp.experts.119.down_proj",
"mtp.layers.0.mlp.experts.119.gate_proj",
"mtp.layers.0.mlp.experts.119.up_proj",
"mtp.layers.0.mlp.experts.12.down_proj",
"mtp.layers.0.mlp.experts.12.gate_proj",
"mtp.layers.0.mlp.experts.12.up_proj",
"mtp.layers.0.mlp.experts.120.down_proj",
"mtp.layers.0.mlp.experts.120.gate_proj",
"mtp.layers.0.mlp.experts.120.up_proj",
"mtp.layers.0.mlp.experts.121.down_proj",
"mtp.layers.0.mlp.experts.121.gate_proj",
"mtp.layers.0.mlp.experts.121.up_proj",
"mtp.layers.0.mlp.experts.122.down_proj",
"mtp.layers.0.mlp.experts.122.gate_proj",
"mtp.layers.0.mlp.experts.122.up_proj",
"mtp.layers.0.mlp.experts.123.down_proj",
"mtp.layers.0.mlp.experts.123.gate_proj",
"mtp.layers.0.mlp.experts.123.up_proj",
"mtp.layers.0.mlp.experts.124.down_proj",
"mtp.layers.0.mlp.experts.124.gate_proj",
"mtp.layers.0.mlp.experts.124.up_proj",
"mtp.layers.0.mlp.experts.125.down_proj",
"mtp.layers.0.mlp.experts.125.gate_proj",
"mtp.layers.0.mlp.experts.125.up_proj",
"mtp.layers.0.mlp.experts.126.down_proj",
"mtp.layers.0.mlp.experts.126.gate_proj",
"mtp.layers.0.mlp.experts.126.up_proj",
"mtp.layers.0.mlp.experts.127.down_proj",
"mtp.layers.0.mlp.experts.127.gate_proj",
"mtp.layers.0.mlp.experts.127.up_proj",
"mtp.layers.0.mlp.experts.128.down_proj",
"mtp.layers.0.mlp.experts.128.gate_proj",
"mtp.layers.0.mlp.experts.128.up_proj",
"mtp.layers.0.mlp.experts.129.down_proj",
"mtp.layers.0.mlp.experts.129.gate_proj",
"mtp.layers.0.mlp.experts.129.up_proj",
"mtp.layers.0.mlp.experts.13.down_proj",
"mtp.layers.0.mlp.experts.13.gate_proj",
"mtp.layers.0.mlp.experts.13.up_proj",
"mtp.layers.0.mlp.experts.130.down_proj",
"mtp.layers.0.mlp.experts.130.gate_proj",
"mtp.layers.0.mlp.experts.130.up_proj",
"mtp.layers.0.mlp.experts.131.down_proj",
"mtp.layers.0.mlp.experts.131.gate_proj",
"mtp.layers.0.mlp.experts.131.up_proj",
"mtp.layers.0.mlp.experts.132.down_proj",
"mtp.layers.0.mlp.experts.132.gate_proj",
"mtp.layers.0.mlp.experts.132.up_proj",
"mtp.layers.0.mlp.experts.133.down_proj",
"mtp.layers.0.mlp.experts.133.gate_proj",
"mtp.layers.0.mlp.experts.133.up_proj",
"mtp.layers.0.mlp.experts.134.down_proj",
"mtp.layers.0.mlp.experts.134.gate_proj",
"mtp.layers.0.mlp.experts.134.up_proj",
"mtp.layers.0.mlp.experts.135.down_proj",
"mtp.layers.0.mlp.experts.135.gate_proj",
"mtp.layers.0.mlp.experts.135.up_proj",
"mtp.layers.0.mlp.experts.136.down_proj",
"mtp.layers.0.mlp.experts.136.gate_proj",
"mtp.layers.0.mlp.experts.136.up_proj",
"mtp.layers.0.mlp.experts.137.down_proj",
"mtp.layers.0.mlp.experts.137.gate_proj",
"mtp.layers.0.mlp.experts.137.up_proj",
"mtp.layers.0.mlp.experts.138.down_proj",
"mtp.layers.0.mlp.experts.138.gate_proj",
"mtp.layers.0.mlp.experts.138.up_proj",
"mtp.layers.0.mlp.experts.139.down_proj",
"mtp.layers.0.mlp.experts.139.gate_proj",
"mtp.layers.0.mlp.experts.139.up_proj",
"mtp.layers.0.mlp.experts.14.down_proj",
"mtp.layers.0.mlp.experts.14.gate_proj",
"mtp.layers.0.mlp.experts.14.up_proj",
"mtp.layers.0.mlp.experts.140.down_proj",
"mtp.layers.0.mlp.experts.140.gate_proj",
"mtp.layers.0.mlp.experts.140.up_proj",
"mtp.layers.0.mlp.experts.141.down_proj",
"mtp.layers.0.mlp.experts.141.gate_proj",
"mtp.layers.0.mlp.experts.141.up_proj",
"mtp.layers.0.mlp.experts.142.down_proj",
"mtp.layers.0.mlp.experts.142.gate_proj",
"mtp.layers.0.mlp.experts.142.up_proj",
"mtp.layers.0.mlp.experts.143.down_proj",
"mtp.layers.0.mlp.experts.143.gate_proj",
"mtp.layers.0.mlp.experts.143.up_proj",
"mtp.layers.0.mlp.experts.144.down_proj",
"mtp.layers.0.mlp.experts.144.gate_proj",
"mtp.layers.0.mlp.experts.144.up_proj",
"mtp.layers.0.mlp.experts.145.down_proj",
"mtp.layers.0.mlp.experts.145.gate_proj",
"mtp.layers.0.mlp.experts.145.up_proj",
"mtp.layers.0.mlp.experts.146.down_proj",
"mtp.layers.0.mlp.experts.146.gate_proj",
"mtp.layers.0.mlp.experts.146.up_proj",
"mtp.layers.0.mlp.experts.147.down_proj",
"mtp.layers.0.mlp.experts.147.gate_proj",
"mtp.layers.0.mlp.experts.147.up_proj",
"mtp.layers.0.mlp.experts.148.down_proj",
"mtp.layers.0.mlp.experts.148.gate_proj",
"mtp.layers.0.mlp.experts.148.up_proj",
"mtp.layers.0.mlp.experts.149.down_proj",
"mtp.layers.0.mlp.experts.149.gate_proj",
"mtp.layers.0.mlp.experts.149.up_proj",
"mtp.layers.0.mlp.experts.15.down_proj",
"mtp.layers.0.mlp.experts.15.gate_proj",
"mtp.layers.0.mlp.experts.15.up_proj",
"mtp.layers.0.mlp.experts.150.down_proj",
"mtp.layers.0.mlp.experts.150.gate_proj",
"mtp.layers.0.mlp.experts.150.up_proj",
"mtp.layers.0.mlp.experts.151.down_proj",
"mtp.layers.0.mlp.experts.151.gate_proj",
"mtp.layers.0.mlp.experts.151.up_proj",
"mtp.layers.0.mlp.experts.152.down_proj",
"mtp.layers.0.mlp.experts.152.gate_proj",
"mtp.layers.0.mlp.experts.152.up_proj",
"mtp.layers.0.mlp.experts.153.down_proj",
"mtp.layers.0.mlp.experts.153.gate_proj",
"mtp.layers.0.mlp.experts.153.up_proj",
"mtp.layers.0.mlp.experts.154.down_proj",
"mtp.layers.0.mlp.experts.154.gate_proj",
"mtp.layers.0.mlp.experts.154.up_proj",
"mtp.layers.0.mlp.experts.155.down_proj",
"mtp.layers.0.mlp.experts.155.gate_proj",
"mtp.layers.0.mlp.experts.155.up_proj",
"mtp.layers.0.mlp.experts.156.down_proj",
"mtp.layers.0.mlp.experts.156.gate_proj",
"mtp.layers.0.mlp.experts.156.up_proj",
"mtp.layers.0.mlp.experts.157.down_proj",
"mtp.layers.0.mlp.experts.157.gate_proj",
"mtp.layers.0.mlp.experts.157.up_proj",
"mtp.layers.0.mlp.experts.158.down_proj",
"mtp.layers.0.mlp.experts.158.gate_proj",
"mtp.layers.0.mlp.experts.158.up_proj",
"mtp.layers.0.mlp.experts.159.down_proj",
"mtp.layers.0.mlp.experts.159.gate_proj",
"mtp.layers.0.mlp.experts.159.up_proj",
"mtp.layers.0.mlp.experts.16.down_proj",
"mtp.layers.0.mlp.experts.16.gate_proj",
"mtp.layers.0.mlp.experts.16.up_proj",
"mtp.layers.0.mlp.experts.160.down_proj",
"mtp.layers.0.mlp.experts.160.gate_proj",
"mtp.layers.0.mlp.experts.160.up_proj",
"mtp.layers.0.mlp.experts.161.down_proj",
"mtp.layers.0.mlp.experts.161.gate_proj",
"mtp.layers.0.mlp.experts.161.up_proj",
"mtp.layers.0.mlp.experts.162.down_proj",
"mtp.layers.0.mlp.experts.162.gate_proj",
"mtp.layers.0.mlp.experts.162.up_proj",
"mtp.layers.0.mlp.experts.163.down_proj",
"mtp.layers.0.mlp.experts.163.gate_proj",
"mtp.layers.0.mlp.experts.163.up_proj",
"mtp.layers.0.mlp.experts.164.down_proj",
"mtp.layers.0.mlp.experts.164.gate_proj",
"mtp.layers.0.mlp.experts.164.up_proj",
"mtp.layers.0.mlp.experts.165.down_proj",
"mtp.layers.0.mlp.experts.165.gate_proj",
"mtp.layers.0.mlp.experts.165.up_proj",
"mtp.layers.0.mlp.experts.166.down_proj",
"mtp.layers.0.mlp.experts.166.gate_proj",
"mtp.layers.0.mlp.experts.166.up_proj",
"mtp.layers.0.mlp.experts.167.down_proj",
"mtp.layers.0.mlp.experts.167.gate_proj",
"mtp.layers.0.mlp.experts.167.up_proj",
"mtp.layers.0.mlp.experts.168.down_proj",
"mtp.layers.0.mlp.experts.168.gate_proj",
"mtp.layers.0.mlp.experts.168.up_proj",
"mtp.layers.0.mlp.experts.169.down_proj",
"mtp.layers.0.mlp.experts.169.gate_proj",
"mtp.layers.0.mlp.experts.169.up_proj",
"mtp.layers.0.mlp.experts.17.down_proj",
"mtp.layers.0.mlp.experts.17.gate_proj",
"mtp.layers.0.mlp.experts.17.up_proj",
"mtp.layers.0.mlp.experts.170.down_proj",
"mtp.layers.0.mlp.experts.170.gate_proj",
"mtp.layers.0.mlp.experts.170.up_proj",
"mtp.layers.0.mlp.experts.171.down_proj",
"mtp.layers.0.mlp.experts.171.gate_proj",
"mtp.layers.0.mlp.experts.171.up_proj",
"mtp.layers.0.mlp.experts.172.down_proj",
"mtp.layers.0.mlp.experts.172.gate_proj",
"mtp.layers.0.mlp.experts.172.up_proj",
"mtp.layers.0.mlp.experts.173.down_proj",
"mtp.layers.0.mlp.experts.173.gate_proj",
"mtp.layers.0.mlp.experts.173.up_proj",
"mtp.layers.0.mlp.experts.174.down_proj",
"mtp.layers.0.mlp.experts.174.gate_proj",
"mtp.layers.0.mlp.experts.174.up_proj",
"mtp.layers.0.mlp.experts.175.down_proj",
"mtp.layers.0.mlp.experts.175.gate_proj",
"mtp.layers.0.mlp.experts.175.up_proj",
"mtp.layers.0.mlp.experts.176.down_proj",
"mtp.layers.0.mlp.experts.176.gate_proj",
"mtp.layers.0.mlp.experts.176.up_proj",
"mtp.layers.0.mlp.experts.177.down_proj",
"mtp.layers.0.mlp.experts.177.gate_proj",
"mtp.layers.0.mlp.experts.177.up_proj",
"mtp.layers.0.mlp.experts.178.down_proj",
"mtp.layers.0.mlp.experts.178.gate_proj",
"mtp.layers.0.mlp.experts.178.up_proj",
"mtp.layers.0.mlp.experts.179.down_proj",
"mtp.layers.0.mlp.experts.179.gate_proj",
"mtp.layers.0.mlp.experts.179.up_proj",
"mtp.layers.0.mlp.experts.18.down_proj",
"mtp.layers.0.mlp.experts.18.gate_proj",
"mtp.layers.0.mlp.experts.18.up_proj",
"mtp.layers.0.mlp.experts.180.down_proj",
"mtp.layers.0.mlp.experts.180.gate_proj",
"mtp.layers.0.mlp.experts.180.up_proj",
"mtp.layers.0.mlp.experts.181.down_proj",
"mtp.layers.0.mlp.experts.181.gate_proj",
"mtp.layers.0.mlp.experts.181.up_proj",
"mtp.layers.0.mlp.experts.182.down_proj",
"mtp.layers.0.mlp.experts.182.gate_proj",
"mtp.layers.0.mlp.experts.182.up_proj",
"mtp.layers.0.mlp.experts.183.down_proj",
"mtp.layers.0.mlp.experts.183.gate_proj",
"mtp.layers.0.mlp.experts.183.up_proj",
"mtp.layers.0.mlp.experts.184.down_proj",
"mtp.layers.0.mlp.experts.184.gate_proj",
"mtp.layers.0.mlp.experts.184.up_proj",
"mtp.layers.0.mlp.experts.185.down_proj",
"mtp.layers.0.mlp.experts.185.gate_proj",
"mtp.layers.0.mlp.experts.185.up_proj",
"mtp.layers.0.mlp.experts.186.down_proj",
"mtp.layers.0.mlp.experts.186.gate_proj",
"mtp.layers.0.mlp.experts.186.up_proj",
"mtp.layers.0.mlp.experts.187.down_proj",
"mtp.layers.0.mlp.experts.187.gate_proj",
"mtp.layers.0.mlp.experts.187.up_proj",
"mtp.layers.0.mlp.experts.188.down_proj",
"mtp.layers.0.mlp.experts.188.gate_proj",
"mtp.layers.0.mlp.experts.188.up_proj",
"mtp.layers.0.mlp.experts.189.down_proj",
"mtp.layers.0.mlp.experts.189.gate_proj",
"mtp.layers.0.mlp.experts.189.up_proj",
"mtp.layers.0.mlp.experts.19.down_proj",
"mtp.layers.0.mlp.experts.19.gate_proj",
"mtp.layers.0.mlp.experts.19.up_proj",
"mtp.layers.0.mlp.experts.190.down_proj",
"mtp.layers.0.mlp.experts.190.gate_proj",
"mtp.layers.0.mlp.experts.190.up_proj",
"mtp.layers.0.mlp.experts.191.down_proj",
"mtp.layers.0.mlp.experts.191.gate_proj",
"mtp.layers.0.mlp.experts.191.up_proj",
"mtp.layers.0.mlp.experts.192.down_proj",
"mtp.layers.0.mlp.experts.192.gate_proj",
"mtp.layers.0.mlp.experts.192.up_proj",
"mtp.layers.0.mlp.experts.193.down_proj",
"mtp.layers.0.mlp.experts.193.gate_proj",
"mtp.layers.0.mlp.experts.193.up_proj",
"mtp.layers.0.mlp.experts.194.down_proj",
"mtp.layers.0.mlp.experts.194.gate_proj",
"mtp.layers.0.mlp.experts.194.up_proj",
"mtp.layers.0.mlp.experts.195.down_proj",
"mtp.layers.0.mlp.experts.195.gate_proj",
"mtp.layers.0.mlp.experts.195.up_proj",
"mtp.layers.0.mlp.experts.196.down_proj",
"mtp.layers.0.mlp.experts.196.gate_proj",
"mtp.layers.0.mlp.experts.196.up_proj",
"mtp.layers.0.mlp.experts.197.down_proj",
"mtp.layers.0.mlp.experts.197.gate_proj",
"mtp.layers.0.mlp.experts.197.up_proj",
"mtp.layers.0.mlp.experts.198.down_proj",
"mtp.layers.0.mlp.experts.198.gate_proj",
"mtp.layers.0.mlp.experts.198.up_proj",
"mtp.layers.0.mlp.experts.199.down_proj",
"mtp.layers.0.mlp.experts.199.gate_proj",
"mtp.layers.0.mlp.experts.199.up_proj",
"mtp.layers.0.mlp.experts.2.down_proj",
"mtp.layers.0.mlp.experts.2.gate_proj",
"mtp.layers.0.mlp.experts.2.up_proj",
"mtp.layers.0.mlp.experts.20.down_proj",
"mtp.layers.0.mlp.experts.20.gate_proj",
"mtp.layers.0.mlp.experts.20.up_proj",
"mtp.layers.0.mlp.experts.200.down_proj",
"mtp.layers.0.mlp.experts.200.gate_proj",
"mtp.layers.0.mlp.experts.200.up_proj",
"mtp.layers.0.mlp.experts.201.down_proj",
"mtp.layers.0.mlp.experts.201.gate_proj",
"mtp.layers.0.mlp.experts.201.up_proj",
"mtp.layers.0.mlp.experts.202.down_proj",
"mtp.layers.0.mlp.experts.202.gate_proj",
"mtp.layers.0.mlp.experts.202.up_proj",
"mtp.layers.0.mlp.experts.203.down_proj",
"mtp.layers.0.mlp.experts.203.gate_proj",
"mtp.layers.0.mlp.experts.203.up_proj",
"mtp.layers.0.mlp.experts.204.down_proj",
"mtp.layers.0.mlp.experts.204.gate_proj",
"mtp.layers.0.mlp.experts.204.up_proj",
"mtp.layers.0.mlp.experts.205.down_proj",
"mtp.layers.0.mlp.experts.205.gate_proj",
"mtp.layers.0.mlp.experts.205.up_proj",
"mtp.layers.0.mlp.experts.206.down_proj",
"mtp.layers.0.mlp.experts.206.gate_proj",
"mtp.layers.0.mlp.experts.206.up_proj",
"mtp.layers.0.mlp.experts.207.down_proj",
"mtp.layers.0.mlp.experts.207.gate_proj",
"mtp.layers.0.mlp.experts.207.up_proj",
"mtp.layers.0.mlp.experts.208.down_proj",
"mtp.layers.0.mlp.experts.208.gate_proj",
"mtp.layers.0.mlp.experts.208.up_proj",
"mtp.layers.0.mlp.experts.209.down_proj",
"mtp.layers.0.mlp.experts.209.gate_proj",
"mtp.layers.0.mlp.experts.209.up_proj",
"mtp.layers.0.mlp.experts.21.down_proj",
"mtp.layers.0.mlp.experts.21.gate_proj",
"mtp.layers.0.mlp.experts.21.up_proj",
"mtp.layers.0.mlp.experts.210.down_proj",
"mtp.layers.0.mlp.experts.210.gate_proj",
"mtp.layers.0.mlp.experts.210.up_proj",
"mtp.layers.0.mlp.experts.211.down_proj",
"mtp.layers.0.mlp.experts.211.gate_proj",
"mtp.layers.0.mlp.experts.211.up_proj",
"mtp.layers.0.mlp.experts.212.down_proj",
"mtp.layers.0.mlp.experts.212.gate_proj",
"mtp.layers.0.mlp.experts.212.up_proj",
"mtp.layers.0.mlp.experts.213.down_proj",
"mtp.layers.0.mlp.experts.213.gate_proj",
"mtp.layers.0.mlp.experts.213.up_proj",
"mtp.layers.0.mlp.experts.214.down_proj",
"mtp.layers.0.mlp.experts.214.gate_proj",
"mtp.layers.0.mlp.experts.214.up_proj",
"mtp.layers.0.mlp.experts.215.down_proj",
"mtp.layers.0.mlp.experts.215.gate_proj",
"mtp.layers.0.mlp.experts.215.up_proj",
"mtp.layers.0.mlp.experts.216.down_proj",
"mtp.layers.0.mlp.experts.216.gate_proj",
"mtp.layers.0.mlp.experts.216.up_proj",
"mtp.layers.0.mlp.experts.217.down_proj",
"mtp.layers.0.mlp.experts.217.gate_proj",
"mtp.layers.0.mlp.experts.217.up_proj",
"mtp.layers.0.mlp.experts.218.down_proj",
"mtp.layers.0.mlp.experts.218.gate_proj",
"mtp.layers.0.mlp.experts.218.up_proj",
"mtp.layers.0.mlp.experts.219.down_proj",
"mtp.layers.0.mlp.experts.219.gate_proj",
"mtp.layers.0.mlp.experts.219.up_proj",
"mtp.layers.0.mlp.experts.22.down_proj",
"mtp.layers.0.mlp.experts.22.gate_proj",
"mtp.layers.0.mlp.experts.22.up_proj",
"mtp.layers.0.mlp.experts.220.down_proj",
"mtp.layers.0.mlp.experts.220.gate_proj",
"mtp.layers.0.mlp.experts.220.up_proj",
"mtp.layers.0.mlp.experts.221.down_proj",
"mtp.layers.0.mlp.experts.221.gate_proj",
"mtp.layers.0.mlp.experts.221.up_proj",
"mtp.layers.0.mlp.experts.222.down_proj",
"mtp.layers.0.mlp.experts.222.gate_proj",
"mtp.layers.0.mlp.experts.222.up_proj",
"mtp.layers.0.mlp.experts.223.down_proj",
"mtp.layers.0.mlp.experts.223.gate_proj",
"mtp.layers.0.mlp.experts.223.up_proj",
"mtp.layers.0.mlp.experts.224.down_proj",
"mtp.layers.0.mlp.experts.224.gate_proj",
"mtp.layers.0.mlp.experts.224.up_proj",
"mtp.layers.0.mlp.experts.225.down_proj",
"mtp.layers.0.mlp.experts.225.gate_proj",
"mtp.layers.0.mlp.experts.225.up_proj",
"mtp.layers.0.mlp.experts.226.down_proj",
"mtp.layers.0.mlp.experts.226.gate_proj",
"mtp.layers.0.mlp.experts.226.up_proj",
"mtp.layers.0.mlp.experts.227.down_proj",
"mtp.layers.0.mlp.experts.227.gate_proj",
"mtp.layers.0.mlp.experts.227.up_proj",
"mtp.layers.0.mlp.experts.228.down_proj",
"mtp.layers.0.mlp.experts.228.gate_proj",
"mtp.layers.0.mlp.experts.228.up_proj",
"mtp.layers.0.mlp.experts.229.down_proj",
"mtp.layers.0.mlp.experts.229.gate_proj",
"mtp.layers.0.mlp.experts.229.up_proj",
"mtp.layers.0.mlp.experts.23.down_proj",
"mtp.layers.0.mlp.experts.23.gate_proj",
"mtp.layers.0.mlp.experts.23.up_proj",
"mtp.layers.0.mlp.experts.230.down_proj",
"mtp.layers.0.mlp.experts.230.gate_proj",
"mtp.layers.0.mlp.experts.230.up_proj",
"mtp.layers.0.mlp.experts.231.down_proj",
"mtp.layers.0.mlp.experts.231.gate_proj",
"mtp.layers.0.mlp.experts.231.up_proj",
"mtp.layers.0.mlp.experts.232.down_proj",
"mtp.layers.0.mlp.experts.232.gate_proj",
"mtp.layers.0.mlp.experts.232.up_proj",
"mtp.layers.0.mlp.experts.233.down_proj",
"mtp.layers.0.mlp.experts.233.gate_proj",
"mtp.layers.0.mlp.experts.233.up_proj",
"mtp.layers.0.mlp.experts.234.down_proj",
"mtp.layers.0.mlp.experts.234.gate_proj",
"mtp.layers.0.mlp.experts.234.up_proj",
"mtp.layers.0.mlp.experts.235.down_proj",
"mtp.layers.0.mlp.experts.235.gate_proj",
"mtp.layers.0.mlp.experts.235.up_proj",
"mtp.layers.0.mlp.experts.236.down_proj",
"mtp.layers.0.mlp.experts.236.gate_proj",
"mtp.layers.0.mlp.experts.236.up_proj",
"mtp.layers.0.mlp.experts.237.down_proj",
"mtp.layers.0.mlp.experts.237.gate_proj",
"mtp.layers.0.mlp.experts.237.up_proj",
"mtp.layers.0.mlp.experts.238.down_proj",
"mtp.layers.0.mlp.experts.238.gate_proj",
"mtp.layers.0.mlp.experts.238.up_proj",
"mtp.layers.0.mlp.experts.239.down_proj",
"mtp.layers.0.mlp.experts.239.gate_proj",
"mtp.layers.0.mlp.experts.239.up_proj",
"mtp.layers.0.mlp.experts.24.down_proj",
"mtp.layers.0.mlp.experts.24.gate_proj",
"mtp.layers.0.mlp.experts.24.up_proj",
"mtp.layers.0.mlp.experts.240.down_proj",
"mtp.layers.0.mlp.experts.240.gate_proj",
"mtp.layers.0.mlp.experts.240.up_proj",
"mtp.layers.0.mlp.experts.241.down_proj",
"mtp.layers.0.mlp.experts.241.gate_proj",
"mtp.layers.0.mlp.experts.241.up_proj",
"mtp.layers.0.mlp.experts.242.down_proj",
"mtp.layers.0.mlp.experts.242.gate_proj",
"mtp.layers.0.mlp.experts.242.up_proj",
"mtp.layers.0.mlp.experts.243.down_proj",
"mtp.layers.0.mlp.experts.243.gate_proj",
"mtp.layers.0.mlp.experts.243.up_proj",
"mtp.layers.0.mlp.experts.244.down_proj",
"mtp.layers.0.mlp.experts.244.gate_proj",
"mtp.layers.0.mlp.experts.244.up_proj",
"mtp.layers.0.mlp.experts.245.down_proj",
"mtp.layers.0.mlp.experts.245.gate_proj",
"mtp.layers.0.mlp.experts.245.up_proj",
"mtp.layers.0.mlp.experts.246.down_proj",
"mtp.layers.0.mlp.experts.246.gate_proj",
"mtp.layers.0.mlp.experts.246.up_proj",
"mtp.layers.0.mlp.experts.247.down_proj",
"mtp.layers.0.mlp.experts.247.gate_proj",
"mtp.layers.0.mlp.experts.247.up_proj",
"mtp.layers.0.mlp.experts.248.down_proj",
"mtp.layers.0.mlp.experts.248.gate_proj",
"mtp.layers.0.mlp.experts.248.up_proj",
"mtp.layers.0.mlp.experts.249.down_proj",
"mtp.layers.0.mlp.experts.249.gate_proj",
"mtp.layers.0.mlp.experts.249.up_proj",
"mtp.layers.0.mlp.experts.25.down_proj",
"mtp.layers.0.mlp.experts.25.gate_proj",
"mtp.layers.0.mlp.experts.25.up_proj",
"mtp.layers.0.mlp.experts.250.down_proj",
"mtp.layers.0.mlp.experts.250.gate_proj",
"mtp.layers.0.mlp.experts.250.up_proj",
"mtp.layers.0.mlp.experts.251.down_proj",
"mtp.layers.0.mlp.experts.251.gate_proj",
"mtp.layers.0.mlp.experts.251.up_proj",
"mtp.layers.0.mlp.experts.252.down_proj",
"mtp.layers.0.mlp.experts.252.gate_proj",
"mtp.layers.0.mlp.experts.252.up_proj",
"mtp.layers.0.mlp.experts.253.down_proj",
"mtp.layers.0.mlp.experts.253.gate_proj",
"mtp.layers.0.mlp.experts.253.up_proj",
"mtp.layers.0.mlp.experts.254.down_proj",
"mtp.layers.0.mlp.experts.254.gate_proj",
"mtp.layers.0.mlp.experts.254.up_proj",
"mtp.layers.0.mlp.experts.255.down_proj",
"mtp.layers.0.mlp.experts.255.gate_proj",
"mtp.layers.0.mlp.experts.255.up_proj",
"mtp.layers.0.mlp.experts.256.down_proj",
"mtp.layers.0.mlp.experts.256.gate_proj",
"mtp.layers.0.mlp.experts.256.up_proj",
"mtp.layers.0.mlp.experts.257.down_proj",
"mtp.layers.0.mlp.experts.257.gate_proj",
"mtp.layers.0.mlp.experts.257.up_proj",
"mtp.layers.0.mlp.experts.258.down_proj",
"mtp.layers.0.mlp.experts.258.gate_proj",
"mtp.layers.0.mlp.experts.258.up_proj",
"mtp.layers.0.mlp.experts.259.down_proj",
"mtp.layers.0.mlp.experts.259.gate_proj",
"mtp.layers.0.mlp.experts.259.up_proj",
"mtp.layers.0.mlp.experts.26.down_proj",
"mtp.layers.0.mlp.experts.26.gate_proj",
"mtp.layers.0.mlp.experts.26.up_proj",
"mtp.layers.0.mlp.experts.260.down_proj",
"mtp.layers.0.mlp.experts.260.gate_proj",
"mtp.layers.0.mlp.experts.260.up_proj",
"mtp.layers.0.mlp.experts.261.down_proj",
"mtp.layers.0.mlp.experts.261.gate_proj",
"mtp.layers.0.mlp.experts.261.up_proj",
"mtp.layers.0.mlp.experts.262.down_proj",
"mtp.layers.0.mlp.experts.262.gate_proj",
"mtp.layers.0.mlp.experts.262.up_proj",
"mtp.layers.0.mlp.experts.263.down_proj",
"mtp.layers.0.mlp.experts.263.gate_proj",
"mtp.layers.0.mlp.experts.263.up_proj",
"mtp.layers.0.mlp.experts.264.down_proj",
"mtp.layers.0.mlp.experts.264.gate_proj",
"mtp.layers.0.mlp.experts.264.up_proj",
"mtp.layers.0.mlp.experts.265.down_proj",
"mtp.layers.0.mlp.experts.265.gate_proj",
"mtp.layers.0.mlp.experts.265.up_proj",
"mtp.layers.0.mlp.experts.266.down_proj",
"mtp.layers.0.mlp.experts.266.gate_proj",
"mtp.layers.0.mlp.experts.266.up_proj",
"mtp.layers.0.mlp.experts.267.down_proj",
"mtp.layers.0.mlp.experts.267.gate_proj",
"mtp.layers.0.mlp.experts.267.up_proj",
"mtp.layers.0.mlp.experts.268.down_proj",
"mtp.layers.0.mlp.experts.268.gate_proj",
"mtp.layers.0.mlp.experts.268.up_proj",
"mtp.layers.0.mlp.experts.269.down_proj",
"mtp.layers.0.mlp.experts.269.gate_proj",
"mtp.layers.0.mlp.experts.269.up_proj",
"mtp.layers.0.mlp.experts.27.down_proj",
"mtp.layers.0.mlp.experts.27.gate_proj",
"mtp.layers.0.mlp.experts.27.up_proj",
"mtp.layers.0.mlp.experts.270.down_proj",
"mtp.layers.0.mlp.experts.270.gate_proj",
"mtp.layers.0.mlp.experts.270.up_proj",
"mtp.layers.0.mlp.experts.271.down_proj",
"mtp.layers.0.mlp.experts.271.gate_proj",
"mtp.layers.0.mlp.experts.271.up_proj",
"mtp.layers.0.mlp.experts.272.down_proj",
"mtp.layers.0.mlp.experts.272.gate_proj",
"mtp.layers.0.mlp.experts.272.up_proj",
"mtp.layers.0.mlp.experts.273.down_proj",
"mtp.layers.0.mlp.experts.273.gate_proj",
"mtp.layers.0.mlp.experts.273.up_proj",
"mtp.layers.0.mlp.experts.274.down_proj",
"mtp.layers.0.mlp.experts.274.gate_proj",
"mtp.layers.0.mlp.experts.274.up_proj",
"mtp.layers.0.mlp.experts.275.down_proj",
"mtp.layers.0.mlp.experts.275.gate_proj",
"mtp.layers.0.mlp.experts.275.up_proj",
"mtp.layers.0.mlp.experts.276.down_proj",
"mtp.layers.0.mlp.experts.276.gate_proj",
"mtp.layers.0.mlp.experts.276.up_proj",
"mtp.layers.0.mlp.experts.277.down_proj",
"mtp.layers.0.mlp.experts.277.gate_proj",
"mtp.layers.0.mlp.experts.277.up_proj",
"mtp.layers.0.mlp.experts.278.down_proj",
"mtp.layers.0.mlp.experts.278.gate_proj",
"mtp.layers.0.mlp.experts.278.up_proj",
"mtp.layers.0.mlp.experts.279.down_proj",
"mtp.layers.0.mlp.experts.279.gate_proj",
"mtp.layers.0.mlp.experts.279.up_proj",
"mtp.layers.0.mlp.experts.28.down_proj",
"mtp.layers.0.mlp.experts.28.gate_proj",
"mtp.layers.0.mlp.experts.28.up_proj",
"mtp.layers.0.mlp.experts.280.down_proj",
"mtp.layers.0.mlp.experts.280.gate_proj",
"mtp.layers.0.mlp.experts.280.up_proj",
"mtp.layers.0.mlp.experts.281.down_proj",
"mtp.layers.0.mlp.experts.281.gate_proj",
"mtp.layers.0.mlp.experts.281.up_proj",
"mtp.layers.0.mlp.experts.282.down_proj",
"mtp.layers.0.mlp.experts.282.gate_proj",
"mtp.layers.0.mlp.experts.282.up_proj",
"mtp.layers.0.mlp.experts.283.down_proj",
"mtp.layers.0.mlp.experts.283.gate_proj",
"mtp.layers.0.mlp.experts.283.up_proj",
"mtp.layers.0.mlp.experts.284.down_proj",
"mtp.layers.0.mlp.experts.284.gate_proj",
"mtp.layers.0.mlp.experts.284.up_proj",
"mtp.layers.0.mlp.experts.285.down_proj",
"mtp.layers.0.mlp.experts.285.gate_proj",
"mtp.layers.0.mlp.experts.285.up_proj",
"mtp.layers.0.mlp.experts.286.down_proj",
"mtp.layers.0.mlp.experts.286.gate_proj",
"mtp.layers.0.mlp.experts.286.up_proj",
"mtp.layers.0.mlp.experts.287.down_proj",
"mtp.layers.0.mlp.experts.287.gate_proj",
"mtp.layers.0.mlp.experts.287.up_proj",
"mtp.layers.0.mlp.experts.288.down_proj",
"mtp.layers.0.mlp.experts.288.gate_proj",
"mtp.layers.0.mlp.experts.288.up_proj",
"mtp.layers.0.mlp.experts.289.down_proj",
"mtp.layers.0.mlp.experts.289.gate_proj",
"mtp.layers.0.mlp.experts.289.up_proj",
"mtp.layers.0.mlp.experts.29.down_proj",
"mtp.layers.0.mlp.experts.29.gate_proj",
"mtp.layers.0.mlp.experts.29.up_proj",
"mtp.layers.0.mlp.experts.290.down_proj",
"mtp.layers.0.mlp.experts.290.gate_proj",
"mtp.layers.0.mlp.experts.290.up_proj",
"mtp.layers.0.mlp.experts.291.down_proj",
"mtp.layers.0.mlp.experts.291.gate_proj",
"mtp.layers.0.mlp.experts.291.up_proj",
"mtp.layers.0.mlp.experts.292.down_proj",
"mtp.layers.0.mlp.experts.292.gate_proj",
"mtp.layers.0.mlp.experts.292.up_proj",
"mtp.layers.0.mlp.experts.293.down_proj",
"mtp.layers.0.mlp.experts.293.gate_proj",
"mtp.layers.0.mlp.experts.293.up_proj",
"mtp.layers.0.mlp.experts.294.down_proj",
"mtp.layers.0.mlp.experts.294.gate_proj",
"mtp.layers.0.mlp.experts.294.up_proj",
"mtp.layers.0.mlp.experts.295.down_proj",
"mtp.layers.0.mlp.experts.295.gate_proj",
"mtp.layers.0.mlp.experts.295.up_proj",
"mtp.layers.0.mlp.experts.296.down_proj",
"mtp.layers.0.mlp.experts.296.gate_proj",
"mtp.layers.0.mlp.experts.296.up_proj",
"mtp.layers.0.mlp.experts.297.down_proj",
"mtp.layers.0.mlp.experts.297.gate_proj",
"mtp.layers.0.mlp.experts.297.up_proj",
"mtp.layers.0.mlp.experts.298.down_proj",
"mtp.layers.0.mlp.experts.298.gate_proj",
"mtp.layers.0.mlp.experts.298.up_proj",
"mtp.layers.0.mlp.experts.299.down_proj",
"mtp.layers.0.mlp.experts.299.gate_proj",
"mtp.layers.0.mlp.experts.299.up_proj",
"mtp.layers.0.mlp.experts.3.down_proj",
"mtp.layers.0.mlp.experts.3.gate_proj",
"mtp.layers.0.mlp.experts.3.up_proj",
"mtp.layers.0.mlp.experts.30.down_proj",
"mtp.layers.0.mlp.experts.30.gate_proj",
"mtp.layers.0.mlp.experts.30.up_proj",
"mtp.layers.0.mlp.experts.300.down_proj",
"mtp.layers.0.mlp.experts.300.gate_proj",
"mtp.layers.0.mlp.experts.300.up_proj",
"mtp.layers.0.mlp.experts.301.down_proj",
"mtp.layers.0.mlp.experts.301.gate_proj",
"mtp.layers.0.mlp.experts.301.up_proj",
"mtp.layers.0.mlp.experts.302.down_proj",
"mtp.layers.0.mlp.experts.302.gate_proj",
"mtp.layers.0.mlp.experts.302.up_proj",
"mtp.layers.0.mlp.experts.303.down_proj",
"mtp.layers.0.mlp.experts.303.gate_proj",
"mtp.layers.0.mlp.experts.303.up_proj",
"mtp.layers.0.mlp.experts.304.down_proj",
"mtp.layers.0.mlp.experts.304.gate_proj",
"mtp.layers.0.mlp.experts.304.up_proj",
"mtp.layers.0.mlp.experts.305.down_proj",
"mtp.layers.0.mlp.experts.305.gate_proj",
"mtp.layers.0.mlp.experts.305.up_proj",
"mtp.layers.0.mlp.experts.306.down_proj",
"mtp.layers.0.mlp.experts.306.gate_proj",
"mtp.layers.0.mlp.experts.306.up_proj",
"mtp.layers.0.mlp.experts.307.down_proj",
"mtp.layers.0.mlp.experts.307.gate_proj",
"mtp.layers.0.mlp.experts.307.up_proj",
"mtp.layers.0.mlp.experts.308.down_proj",
"mtp.layers.0.mlp.experts.308.gate_proj",
"mtp.layers.0.mlp.experts.308.up_proj",
"mtp.layers.0.mlp.experts.309.down_proj",
"mtp.layers.0.mlp.experts.309.gate_proj",
"mtp.layers.0.mlp.experts.309.up_proj",
"mtp.layers.0.mlp.experts.31.down_proj",
"mtp.layers.0.mlp.experts.31.gate_proj",
"mtp.layers.0.mlp.experts.31.up_proj",
"mtp.layers.0.mlp.experts.310.down_proj",
"mtp.layers.0.mlp.experts.310.gate_proj",
"mtp.layers.0.mlp.experts.310.up_proj",
"mtp.layers.0.mlp.experts.311.down_proj",
"mtp.layers.0.mlp.experts.311.gate_proj",
"mtp.layers.0.mlp.experts.311.up_proj",
"mtp.layers.0.mlp.experts.312.down_proj",
"mtp.layers.0.mlp.experts.312.gate_proj",
"mtp.layers.0.mlp.experts.312.up_proj",
"mtp.layers.0.mlp.experts.313.down_proj",
"mtp.layers.0.mlp.experts.313.gate_proj",
"mtp.layers.0.mlp.experts.313.up_proj",
"mtp.layers.0.mlp.experts.314.down_proj",
"mtp.layers.0.mlp.experts.314.gate_proj",
"mtp.layers.0.mlp.experts.314.up_proj",
"mtp.layers.0.mlp.experts.315.down_proj",
"mtp.layers.0.mlp.experts.315.gate_proj",
"mtp.layers.0.mlp.experts.315.up_proj",
"mtp.layers.0.mlp.experts.316.down_proj",
"mtp.layers.0.mlp.experts.316.gate_proj",
"mtp.layers.0.mlp.experts.316.up_proj",
"mtp.layers.0.mlp.experts.317.down_proj",
"mtp.layers.0.mlp.experts.317.gate_proj",
"mtp.layers.0.mlp.experts.317.up_proj",
"mtp.layers.0.mlp.experts.318.down_proj",
"mtp.layers.0.mlp.experts.318.gate_proj",
"mtp.layers.0.mlp.experts.318.up_proj",
"mtp.layers.0.mlp.experts.319.down_proj",
"mtp.layers.0.mlp.experts.319.gate_proj",
"mtp.layers.0.mlp.experts.319.up_proj",
"mtp.layers.0.mlp.experts.32.down_proj",
"mtp.layers.0.mlp.experts.32.gate_proj",
"mtp.layers.0.mlp.experts.32.up_proj",
"mtp.layers.0.mlp.experts.320.down_proj",
"mtp.layers.0.mlp.experts.320.gate_proj",
"mtp.layers.0.mlp.experts.320.up_proj",
"mtp.layers.0.mlp.experts.321.down_proj",
"mtp.layers.0.mlp.experts.321.gate_proj",
"mtp.layers.0.mlp.experts.321.up_proj",
"mtp.layers.0.mlp.experts.322.down_proj",
"mtp.layers.0.mlp.experts.322.gate_proj",
"mtp.layers.0.mlp.experts.322.up_proj",
"mtp.layers.0.mlp.experts.323.down_proj",
"mtp.layers.0.mlp.experts.323.gate_proj",
"mtp.layers.0.mlp.experts.323.up_proj",
"mtp.layers.0.mlp.experts.324.down_proj",
"mtp.layers.0.mlp.experts.324.gate_proj",
"mtp.layers.0.mlp.experts.324.up_proj",
"mtp.layers.0.mlp.experts.325.down_proj",
"mtp.layers.0.mlp.experts.325.gate_proj",
"mtp.layers.0.mlp.experts.325.up_proj",
"mtp.layers.0.mlp.experts.326.down_proj",
"mtp.layers.0.mlp.experts.326.gate_proj",
"mtp.layers.0.mlp.experts.326.up_proj",
"mtp.layers.0.mlp.experts.327.down_proj",
"mtp.layers.0.mlp.experts.327.gate_proj",
"mtp.layers.0.mlp.experts.327.up_proj",
"mtp.layers.0.mlp.experts.328.down_proj",
"mtp.layers.0.mlp.experts.328.gate_proj",
"mtp.layers.0.mlp.experts.328.up_proj",
"mtp.layers.0.mlp.experts.329.down_proj",
"mtp.layers.0.mlp.experts.329.gate_proj",
"mtp.layers.0.mlp.experts.329.up_proj",
"mtp.layers.0.mlp.experts.33.down_proj",
"mtp.layers.0.mlp.experts.33.gate_proj",
"mtp.layers.0.mlp.experts.33.up_proj",
"mtp.layers.0.mlp.experts.330.down_proj",
"mtp.layers.0.mlp.experts.330.gate_proj",
"mtp.layers.0.mlp.experts.330.up_proj",
"mtp.layers.0.mlp.experts.331.down_proj",
"mtp.layers.0.mlp.experts.331.gate_proj",
"mtp.layers.0.mlp.experts.331.up_proj",
"mtp.layers.0.mlp.experts.332.down_proj",
"mtp.layers.0.mlp.experts.332.gate_proj",
"mtp.layers.0.mlp.experts.332.up_proj",
"mtp.layers.0.mlp.experts.333.down_proj",
"mtp.layers.0.mlp.experts.333.gate_proj",
"mtp.layers.0.mlp.experts.333.up_proj",
"mtp.layers.0.mlp.experts.334.down_proj",
"mtp.layers.0.mlp.experts.334.gate_proj",
"mtp.layers.0.mlp.experts.334.up_proj",
"mtp.layers.0.mlp.experts.335.down_proj",
"mtp.layers.0.mlp.experts.335.gate_proj",
"mtp.layers.0.mlp.experts.335.up_proj",
"mtp.layers.0.mlp.experts.336.down_proj",
"mtp.layers.0.mlp.experts.336.gate_proj",
"mtp.layers.0.mlp.experts.336.up_proj",
"mtp.layers.0.mlp.experts.337.down_proj",
"mtp.layers.0.mlp.experts.337.gate_proj",
"mtp.layers.0.mlp.experts.337.up_proj",
"mtp.layers.0.mlp.experts.338.down_proj",
"mtp.layers.0.mlp.experts.338.gate_proj",
"mtp.layers.0.mlp.experts.338.up_proj",
"mtp.layers.0.mlp.experts.339.down_proj",
"mtp.layers.0.mlp.experts.339.gate_proj",
"mtp.layers.0.mlp.experts.339.up_proj",
"mtp.layers.0.mlp.experts.34.down_proj",
"mtp.layers.0.mlp.experts.34.gate_proj",
"mtp.layers.0.mlp.experts.34.up_proj",
"mtp.layers.0.mlp.experts.340.down_proj",
"mtp.layers.0.mlp.experts.340.gate_proj",
"mtp.layers.0.mlp.experts.340.up_proj",
"mtp.layers.0.mlp.experts.341.down_proj",
"mtp.layers.0.mlp.experts.341.gate_proj",
"mtp.layers.0.mlp.experts.341.up_proj",
"mtp.layers.0.mlp.experts.342.down_proj",
"mtp.layers.0.mlp.experts.342.gate_proj",
"mtp.layers.0.mlp.experts.342.up_proj",
"mtp.layers.0.mlp.experts.343.down_proj",
"mtp.layers.0.mlp.experts.343.gate_proj",
"mtp.layers.0.mlp.experts.343.up_proj",
"mtp.layers.0.mlp.experts.344.down_proj",
"mtp.layers.0.mlp.experts.344.gate_proj",
"mtp.layers.0.mlp.experts.344.up_proj",
"mtp.layers.0.mlp.experts.345.down_proj",
"mtp.layers.0.mlp.experts.345.gate_proj",
"mtp.layers.0.mlp.experts.345.up_proj",
"mtp.layers.0.mlp.experts.346.down_proj",
"mtp.layers.0.mlp.experts.346.gate_proj",
"mtp.layers.0.mlp.experts.346.up_proj",
"mtp.layers.0.mlp.experts.347.down_proj",
"mtp.layers.0.mlp.experts.347.gate_proj",
"mtp.layers.0.mlp.experts.347.up_proj",
"mtp.layers.0.mlp.experts.348.down_proj",
"mtp.layers.0.mlp.experts.348.gate_proj",
"mtp.layers.0.mlp.experts.348.up_proj",
"mtp.layers.0.mlp.experts.349.down_proj",
"mtp.layers.0.mlp.experts.349.gate_proj",
"mtp.layers.0.mlp.experts.349.up_proj",
"mtp.layers.0.mlp.experts.35.down_proj",
"mtp.layers.0.mlp.experts.35.gate_proj",
"mtp.layers.0.mlp.experts.35.up_proj",
"mtp.layers.0.mlp.experts.350.down_proj",
"mtp.layers.0.mlp.experts.350.gate_proj",
"mtp.layers.0.mlp.experts.350.up_proj",
"mtp.layers.0.mlp.experts.351.down_proj",
"mtp.layers.0.mlp.experts.351.gate_proj",
"mtp.layers.0.mlp.experts.351.up_proj",
"mtp.layers.0.mlp.experts.352.down_proj",
"mtp.layers.0.mlp.experts.352.gate_proj",
"mtp.layers.0.mlp.experts.352.up_proj",
"mtp.layers.0.mlp.experts.353.down_proj",
"mtp.layers.0.mlp.experts.353.gate_proj",
"mtp.layers.0.mlp.experts.353.up_proj",
"mtp.layers.0.mlp.experts.354.down_proj",
"mtp.layers.0.mlp.experts.354.gate_proj",
"mtp.layers.0.mlp.experts.354.up_proj",
"mtp.layers.0.mlp.experts.355.down_proj",
"mtp.layers.0.mlp.experts.355.gate_proj",
"mtp.layers.0.mlp.experts.355.up_proj",
"mtp.layers.0.mlp.experts.356.down_proj",
"mtp.layers.0.mlp.experts.356.gate_proj",
"mtp.layers.0.mlp.experts.356.up_proj",
"mtp.layers.0.mlp.experts.357.down_proj",
"mtp.layers.0.mlp.experts.357.gate_proj",
"mtp.layers.0.mlp.experts.357.up_proj",
"mtp.layers.0.mlp.experts.358.down_proj",
"mtp.layers.0.mlp.experts.358.gate_proj",
"mtp.layers.0.mlp.experts.358.up_proj",
"mtp.layers.0.mlp.experts.359.down_proj",
"mtp.layers.0.mlp.experts.359.gate_proj",
"mtp.layers.0.mlp.experts.359.up_proj",
"mtp.layers.0.mlp.experts.36.down_proj",
"mtp.layers.0.mlp.experts.36.gate_proj",
"mtp.layers.0.mlp.experts.36.up_proj",
"mtp.layers.0.mlp.experts.360.down_proj",
"mtp.layers.0.mlp.experts.360.gate_proj",
"mtp.layers.0.mlp.experts.360.up_proj",
"mtp.layers.0.mlp.experts.361.down_proj",
"mtp.layers.0.mlp.experts.361.gate_proj",
"mtp.layers.0.mlp.experts.361.up_proj",
"mtp.layers.0.mlp.experts.362.down_proj",
"mtp.layers.0.mlp.experts.362.gate_proj",
"mtp.layers.0.mlp.experts.362.up_proj",
"mtp.layers.0.mlp.experts.363.down_proj",
"mtp.layers.0.mlp.experts.363.gate_proj",
"mtp.layers.0.mlp.experts.363.up_proj",
"mtp.layers.0.mlp.experts.364.down_proj",
"mtp.layers.0.mlp.experts.364.gate_proj",
"mtp.layers.0.mlp.experts.364.up_proj",
"mtp.layers.0.mlp.experts.365.down_proj",
"mtp.layers.0.mlp.experts.365.gate_proj",
"mtp.layers.0.mlp.experts.365.up_proj",
"mtp.layers.0.mlp.experts.366.down_proj",
"mtp.layers.0.mlp.experts.366.gate_proj",
"mtp.layers.0.mlp.experts.366.up_proj",
"mtp.layers.0.mlp.experts.367.down_proj",
"mtp.layers.0.mlp.experts.367.gate_proj",
"mtp.layers.0.mlp.experts.367.up_proj",
"mtp.layers.0.mlp.experts.368.down_proj",
"mtp.layers.0.mlp.experts.368.gate_proj",
"mtp.layers.0.mlp.experts.368.up_proj",
"mtp.layers.0.mlp.experts.369.down_proj",
"mtp.layers.0.mlp.experts.369.gate_proj",
"mtp.layers.0.mlp.experts.369.up_proj",
"mtp.layers.0.mlp.experts.37.down_proj",
"mtp.layers.0.mlp.experts.37.gate_proj",
"mtp.layers.0.mlp.experts.37.up_proj",
"mtp.layers.0.mlp.experts.370.down_proj",
"mtp.layers.0.mlp.experts.370.gate_proj",
"mtp.layers.0.mlp.experts.370.up_proj",
"mtp.layers.0.mlp.experts.371.down_proj",
"mtp.layers.0.mlp.experts.371.gate_proj",
"mtp.layers.0.mlp.experts.371.up_proj",
"mtp.layers.0.mlp.experts.372.down_proj",
"mtp.layers.0.mlp.experts.372.gate_proj",
"mtp.layers.0.mlp.experts.372.up_proj",
"mtp.layers.0.mlp.experts.373.down_proj",
"mtp.layers.0.mlp.experts.373.gate_proj",
"mtp.layers.0.mlp.experts.373.up_proj",
"mtp.layers.0.mlp.experts.374.down_proj",
"mtp.layers.0.mlp.experts.374.gate_proj",
"mtp.layers.0.mlp.experts.374.up_proj",
"mtp.layers.0.mlp.experts.375.down_proj",
"mtp.layers.0.mlp.experts.375.gate_proj",
"mtp.layers.0.mlp.experts.375.up_proj",
"mtp.layers.0.mlp.experts.376.down_proj",
"mtp.layers.0.mlp.experts.376.gate_proj",
"mtp.layers.0.mlp.experts.376.up_proj",
"mtp.layers.0.mlp.experts.377.down_proj",
"mtp.layers.0.mlp.experts.377.gate_proj",
"mtp.layers.0.mlp.experts.377.up_proj",
"mtp.layers.0.mlp.experts.378.down_proj",
"mtp.layers.0.mlp.experts.378.gate_proj",
"mtp.layers.0.mlp.experts.378.up_proj",
"mtp.layers.0.mlp.experts.379.down_proj",
"mtp.layers.0.mlp.experts.379.gate_proj",
"mtp.layers.0.mlp.experts.379.up_proj",
"mtp.layers.0.mlp.experts.38.down_proj",
"mtp.layers.0.mlp.experts.38.gate_proj",
"mtp.layers.0.mlp.experts.38.up_proj",
"mtp.layers.0.mlp.experts.380.down_proj",
"mtp.layers.0.mlp.experts.380.gate_proj",
"mtp.layers.0.mlp.experts.380.up_proj",
"mtp.layers.0.mlp.experts.381.down_proj",
"mtp.layers.0.mlp.experts.381.gate_proj",
"mtp.layers.0.mlp.experts.381.up_proj",
"mtp.layers.0.mlp.experts.382.down_proj",
"mtp.layers.0.mlp.experts.382.gate_proj",
"mtp.layers.0.mlp.experts.382.up_proj",
"mtp.layers.0.mlp.experts.383.down_proj",
"mtp.layers.0.mlp.experts.383.gate_proj",
"mtp.layers.0.mlp.experts.383.up_proj",
"mtp.layers.0.mlp.experts.384.down_proj",
"mtp.layers.0.mlp.experts.384.gate_proj",
"mtp.layers.0.mlp.experts.384.up_proj",
"mtp.layers.0.mlp.experts.385.down_proj",
"mtp.layers.0.mlp.experts.385.gate_proj",
"mtp.layers.0.mlp.experts.385.up_proj",
"mtp.layers.0.mlp.experts.386.down_proj",
"mtp.layers.0.mlp.experts.386.gate_proj",
"mtp.layers.0.mlp.experts.386.up_proj",
"mtp.layers.0.mlp.experts.387.down_proj",
"mtp.layers.0.mlp.experts.387.gate_proj",
"mtp.layers.0.mlp.experts.387.up_proj",
"mtp.layers.0.mlp.experts.388.down_proj",
"mtp.layers.0.mlp.experts.388.gate_proj",
"mtp.layers.0.mlp.experts.388.up_proj",
"mtp.layers.0.mlp.experts.389.down_proj",
"mtp.layers.0.mlp.experts.389.gate_proj",
"mtp.layers.0.mlp.experts.389.up_proj",
"mtp.layers.0.mlp.experts.39.down_proj",
"mtp.layers.0.mlp.experts.39.gate_proj",
"mtp.layers.0.mlp.experts.39.up_proj",
"mtp.layers.0.mlp.experts.390.down_proj",
"mtp.layers.0.mlp.experts.390.gate_proj",
"mtp.layers.0.mlp.experts.390.up_proj",
"mtp.layers.0.mlp.experts.391.down_proj",
"mtp.layers.0.mlp.experts.391.gate_proj",
"mtp.layers.0.mlp.experts.391.up_proj",
"mtp.layers.0.mlp.experts.392.down_proj",
"mtp.layers.0.mlp.experts.392.gate_proj",
"mtp.layers.0.mlp.experts.392.up_proj",
"mtp.layers.0.mlp.experts.393.down_proj",
"mtp.layers.0.mlp.experts.393.gate_proj",
"mtp.layers.0.mlp.experts.393.up_proj",
"mtp.layers.0.mlp.experts.394.down_proj",
"mtp.layers.0.mlp.experts.394.gate_proj",
"mtp.layers.0.mlp.experts.394.up_proj",
"mtp.layers.0.mlp.experts.395.down_proj",
"mtp.layers.0.mlp.experts.395.gate_proj",
"mtp.layers.0.mlp.experts.395.up_proj",
"mtp.layers.0.mlp.experts.396.down_proj",
"mtp.layers.0.mlp.experts.396.gate_proj",
"mtp.layers.0.mlp.experts.396.up_proj",
"mtp.layers.0.mlp.experts.397.down_proj",
"mtp.layers.0.mlp.experts.397.gate_proj",
"mtp.layers.0.mlp.experts.397.up_proj",
"mtp.layers.0.mlp.experts.398.down_proj",
"mtp.layers.0.mlp.experts.398.gate_proj",
"mtp.layers.0.mlp.experts.398.up_proj",
"mtp.layers.0.mlp.experts.399.down_proj",
"mtp.layers.0.mlp.experts.399.gate_proj",
"mtp.layers.0.mlp.experts.399.up_proj",
"mtp.layers.0.mlp.experts.4.down_proj",
"mtp.layers.0.mlp.experts.4.gate_proj",
"mtp.layers.0.mlp.experts.4.up_proj",
"mtp.layers.0.mlp.experts.40.down_proj",
"mtp.layers.0.mlp.experts.40.gate_proj",
"mtp.layers.0.mlp.experts.40.up_proj",
"mtp.layers.0.mlp.experts.400.down_proj",
"mtp.layers.0.mlp.experts.400.gate_proj",
"mtp.layers.0.mlp.experts.400.up_proj",
"mtp.layers.0.mlp.experts.401.down_proj",
"mtp.layers.0.mlp.experts.401.gate_proj",
"mtp.layers.0.mlp.experts.401.up_proj",
"mtp.layers.0.mlp.experts.402.down_proj",
"mtp.layers.0.mlp.experts.402.gate_proj",
"mtp.layers.0.mlp.experts.402.up_proj",
"mtp.layers.0.mlp.experts.403.down_proj",
"mtp.layers.0.mlp.experts.403.gate_proj",
"mtp.layers.0.mlp.experts.403.up_proj",
"mtp.layers.0.mlp.experts.404.down_proj",
"mtp.layers.0.mlp.experts.404.gate_proj",
"mtp.layers.0.mlp.experts.404.up_proj",
"mtp.layers.0.mlp.experts.405.down_proj",
"mtp.layers.0.mlp.experts.405.gate_proj",
"mtp.layers.0.mlp.experts.405.up_proj",
"mtp.layers.0.mlp.experts.406.down_proj",
"mtp.layers.0.mlp.experts.406.gate_proj",
"mtp.layers.0.mlp.experts.406.up_proj",
"mtp.layers.0.mlp.experts.407.down_proj",
"mtp.layers.0.mlp.experts.407.gate_proj",
"mtp.layers.0.mlp.experts.407.up_proj",
"mtp.layers.0.mlp.experts.408.down_proj",
"mtp.layers.0.mlp.experts.408.gate_proj",
"mtp.layers.0.mlp.experts.408.up_proj",
"mtp.layers.0.mlp.experts.409.down_proj",
"mtp.layers.0.mlp.experts.409.gate_proj",
"mtp.layers.0.mlp.experts.409.up_proj",
"mtp.layers.0.mlp.experts.41.down_proj",
"mtp.layers.0.mlp.experts.41.gate_proj",
"mtp.layers.0.mlp.experts.41.up_proj",
"mtp.layers.0.mlp.experts.410.down_proj",
"mtp.layers.0.mlp.experts.410.gate_proj",
"mtp.layers.0.mlp.experts.410.up_proj",
"mtp.layers.0.mlp.experts.411.down_proj",
"mtp.layers.0.mlp.experts.411.gate_proj",
"mtp.layers.0.mlp.experts.411.up_proj",
"mtp.layers.0.mlp.experts.412.down_proj",
"mtp.layers.0.mlp.experts.412.gate_proj",
"mtp.layers.0.mlp.experts.412.up_proj",
"mtp.layers.0.mlp.experts.413.down_proj",
"mtp.layers.0.mlp.experts.413.gate_proj",
"mtp.layers.0.mlp.experts.413.up_proj",
"mtp.layers.0.mlp.experts.414.down_proj",
"mtp.layers.0.mlp.experts.414.gate_proj",
"mtp.layers.0.mlp.experts.414.up_proj",
"mtp.layers.0.mlp.experts.415.down_proj",
"mtp.layers.0.mlp.experts.415.gate_proj",
"mtp.layers.0.mlp.experts.415.up_proj",
"mtp.layers.0.mlp.experts.416.down_proj",
"mtp.layers.0.mlp.experts.416.gate_proj",
"mtp.layers.0.mlp.experts.416.up_proj",
"mtp.layers.0.mlp.experts.417.down_proj",
"mtp.layers.0.mlp.experts.417.gate_proj",
"mtp.layers.0.mlp.experts.417.up_proj",
"mtp.layers.0.mlp.experts.418.down_proj",
"mtp.layers.0.mlp.experts.418.gate_proj",
"mtp.layers.0.mlp.experts.418.up_proj",
"mtp.layers.0.mlp.experts.419.down_proj",
"mtp.layers.0.mlp.experts.419.gate_proj",
"mtp.layers.0.mlp.experts.419.up_proj",
"mtp.layers.0.mlp.experts.42.down_proj",
"mtp.layers.0.mlp.experts.42.gate_proj",
"mtp.layers.0.mlp.experts.42.up_proj",
"mtp.layers.0.mlp.experts.420.down_proj",
"mtp.layers.0.mlp.experts.420.gate_proj",
"mtp.layers.0.mlp.experts.420.up_proj",
"mtp.layers.0.mlp.experts.421.down_proj",
"mtp.layers.0.mlp.experts.421.gate_proj",
"mtp.layers.0.mlp.experts.421.up_proj",
"mtp.layers.0.mlp.experts.422.down_proj",
"mtp.layers.0.mlp.experts.422.gate_proj",
"mtp.layers.0.mlp.experts.422.up_proj",
"mtp.layers.0.mlp.experts.423.down_proj",
"mtp.layers.0.mlp.experts.423.gate_proj",
"mtp.layers.0.mlp.experts.423.up_proj",
"mtp.layers.0.mlp.experts.424.down_proj",
"mtp.layers.0.mlp.experts.424.gate_proj",
"mtp.layers.0.mlp.experts.424.up_proj",
"mtp.layers.0.mlp.experts.425.down_proj",
"mtp.layers.0.mlp.experts.425.gate_proj",
"mtp.layers.0.mlp.experts.425.up_proj",
"mtp.layers.0.mlp.experts.426.down_proj",
"mtp.layers.0.mlp.experts.426.gate_proj",
"mtp.layers.0.mlp.experts.426.up_proj",
"mtp.layers.0.mlp.experts.427.down_proj",
"mtp.layers.0.mlp.experts.427.gate_proj",
"mtp.layers.0.mlp.experts.427.up_proj",
"mtp.layers.0.mlp.experts.428.down_proj",
"mtp.layers.0.mlp.experts.428.gate_proj",
"mtp.layers.0.mlp.experts.428.up_proj",
"mtp.layers.0.mlp.experts.429.down_proj",
"mtp.layers.0.mlp.experts.429.gate_proj",
"mtp.layers.0.mlp.experts.429.up_proj",
"mtp.layers.0.mlp.experts.43.down_proj",
"mtp.layers.0.mlp.experts.43.gate_proj",
"mtp.layers.0.mlp.experts.43.up_proj",
"mtp.layers.0.mlp.experts.430.down_proj",
"mtp.layers.0.mlp.experts.430.gate_proj",
"mtp.layers.0.mlp.experts.430.up_proj",
"mtp.layers.0.mlp.experts.431.down_proj",
"mtp.layers.0.mlp.experts.431.gate_proj",
"mtp.layers.0.mlp.experts.431.up_proj",
"mtp.layers.0.mlp.experts.432.down_proj",
"mtp.layers.0.mlp.experts.432.gate_proj",
"mtp.layers.0.mlp.experts.432.up_proj",
"mtp.layers.0.mlp.experts.433.down_proj",
"mtp.layers.0.mlp.experts.433.gate_proj",
"mtp.layers.0.mlp.experts.433.up_proj",
"mtp.layers.0.mlp.experts.434.down_proj",
"mtp.layers.0.mlp.experts.434.gate_proj",
"mtp.layers.0.mlp.experts.434.up_proj",
"mtp.layers.0.mlp.experts.435.down_proj",
"mtp.layers.0.mlp.experts.435.gate_proj",
"mtp.layers.0.mlp.experts.435.up_proj",
"mtp.layers.0.mlp.experts.436.down_proj",
"mtp.layers.0.mlp.experts.436.gate_proj",
"mtp.layers.0.mlp.experts.436.up_proj",
"mtp.layers.0.mlp.experts.437.down_proj",
"mtp.layers.0.mlp.experts.437.gate_proj",
"mtp.layers.0.mlp.experts.437.up_proj",
"mtp.layers.0.mlp.experts.438.down_proj",
"mtp.layers.0.mlp.experts.438.gate_proj",
"mtp.layers.0.mlp.experts.438.up_proj",
"mtp.layers.0.mlp.experts.439.down_proj",
"mtp.layers.0.mlp.experts.439.gate_proj",
"mtp.layers.0.mlp.experts.439.up_proj",
"mtp.layers.0.mlp.experts.44.down_proj",
"mtp.layers.0.mlp.experts.44.gate_proj",
"mtp.layers.0.mlp.experts.44.up_proj",
"mtp.layers.0.mlp.experts.440.down_proj",
"mtp.layers.0.mlp.experts.440.gate_proj",
"mtp.layers.0.mlp.experts.440.up_proj",
"mtp.layers.0.mlp.experts.441.down_proj",
"mtp.layers.0.mlp.experts.441.gate_proj",
"mtp.layers.0.mlp.experts.441.up_proj",
"mtp.layers.0.mlp.experts.442.down_proj",
"mtp.layers.0.mlp.experts.442.gate_proj",
"mtp.layers.0.mlp.experts.442.up_proj",
"mtp.layers.0.mlp.experts.443.down_proj",
"mtp.layers.0.mlp.experts.443.gate_proj",
"mtp.layers.0.mlp.experts.443.up_proj",
"mtp.layers.0.mlp.experts.444.down_proj",
"mtp.layers.0.mlp.experts.444.gate_proj",
"mtp.layers.0.mlp.experts.444.up_proj",
"mtp.layers.0.mlp.experts.445.down_proj",
"mtp.layers.0.mlp.experts.445.gate_proj",
"mtp.layers.0.mlp.experts.445.up_proj",
"mtp.layers.0.mlp.experts.446.down_proj",
"mtp.layers.0.mlp.experts.446.gate_proj",
"mtp.layers.0.mlp.experts.446.up_proj",
"mtp.layers.0.mlp.experts.447.down_proj",
"mtp.layers.0.mlp.experts.447.gate_proj",
"mtp.layers.0.mlp.experts.447.up_proj",
"mtp.layers.0.mlp.experts.448.down_proj",
"mtp.layers.0.mlp.experts.448.gate_proj",
"mtp.layers.0.mlp.experts.448.up_proj",
"mtp.layers.0.mlp.experts.449.down_proj",
"mtp.layers.0.mlp.experts.449.gate_proj",
"mtp.layers.0.mlp.experts.449.up_proj",
"mtp.layers.0.mlp.experts.45.down_proj",
"mtp.layers.0.mlp.experts.45.gate_proj",
"mtp.layers.0.mlp.experts.45.up_proj",
"mtp.layers.0.mlp.experts.450.down_proj",
"mtp.layers.0.mlp.experts.450.gate_proj",
"mtp.layers.0.mlp.experts.450.up_proj",
"mtp.layers.0.mlp.experts.451.down_proj",
"mtp.layers.0.mlp.experts.451.gate_proj",
"mtp.layers.0.mlp.experts.451.up_proj",
"mtp.layers.0.mlp.experts.452.down_proj",
"mtp.layers.0.mlp.experts.452.gate_proj",
"mtp.layers.0.mlp.experts.452.up_proj",
"mtp.layers.0.mlp.experts.453.down_proj",
"mtp.layers.0.mlp.experts.453.gate_proj",
"mtp.layers.0.mlp.experts.453.up_proj",
"mtp.layers.0.mlp.experts.454.down_proj",
"mtp.layers.0.mlp.experts.454.gate_proj",
"mtp.layers.0.mlp.experts.454.up_proj",
"mtp.layers.0.mlp.experts.455.down_proj",
"mtp.layers.0.mlp.experts.455.gate_proj",
"mtp.layers.0.mlp.experts.455.up_proj",
"mtp.layers.0.mlp.experts.456.down_proj",
"mtp.layers.0.mlp.experts.456.gate_proj",
"mtp.layers.0.mlp.experts.456.up_proj",
"mtp.layers.0.mlp.experts.457.down_proj",
"mtp.layers.0.mlp.experts.457.gate_proj",
"mtp.layers.0.mlp.experts.457.up_proj",
"mtp.layers.0.mlp.experts.458.down_proj",
"mtp.layers.0.mlp.experts.458.gate_proj",
"mtp.layers.0.mlp.experts.458.up_proj",
"mtp.layers.0.mlp.experts.459.down_proj",
"mtp.layers.0.mlp.experts.459.gate_proj",
"mtp.layers.0.mlp.experts.459.up_proj",
"mtp.layers.0.mlp.experts.46.down_proj",
"mtp.layers.0.mlp.experts.46.gate_proj",
"mtp.layers.0.mlp.experts.46.up_proj",
"mtp.layers.0.mlp.experts.460.down_proj",
"mtp.layers.0.mlp.experts.460.gate_proj",
"mtp.layers.0.mlp.experts.460.up_proj",
"mtp.layers.0.mlp.experts.461.down_proj",
"mtp.layers.0.mlp.experts.461.gate_proj",
"mtp.layers.0.mlp.experts.461.up_proj",
"mtp.layers.0.mlp.experts.462.down_proj",
"mtp.layers.0.mlp.experts.462.gate_proj",
"mtp.layers.0.mlp.experts.462.up_proj",
"mtp.layers.0.mlp.experts.463.down_proj",
"mtp.layers.0.mlp.experts.463.gate_proj",
"mtp.layers.0.mlp.experts.463.up_proj",
"mtp.layers.0.mlp.experts.464.down_proj",
"mtp.layers.0.mlp.experts.464.gate_proj",
"mtp.layers.0.mlp.experts.464.up_proj",
"mtp.layers.0.mlp.experts.465.down_proj",
"mtp.layers.0.mlp.experts.465.gate_proj",
"mtp.layers.0.mlp.experts.465.up_proj",
"mtp.layers.0.mlp.experts.466.down_proj",
"mtp.layers.0.mlp.experts.466.gate_proj",
"mtp.layers.0.mlp.experts.466.up_proj",
"mtp.layers.0.mlp.experts.467.down_proj",
"mtp.layers.0.mlp.experts.467.gate_proj",
"mtp.layers.0.mlp.experts.467.up_proj",
"mtp.layers.0.mlp.experts.468.down_proj",
"mtp.layers.0.mlp.experts.468.gate_proj",
"mtp.layers.0.mlp.experts.468.up_proj",
"mtp.layers.0.mlp.experts.469.down_proj",
"mtp.layers.0.mlp.experts.469.gate_proj",
"mtp.layers.0.mlp.experts.469.up_proj",
"mtp.layers.0.mlp.experts.47.down_proj",
"mtp.layers.0.mlp.experts.47.gate_proj",
"mtp.layers.0.mlp.experts.47.up_proj",
"mtp.layers.0.mlp.experts.470.down_proj",
"mtp.layers.0.mlp.experts.470.gate_proj",
"mtp.layers.0.mlp.experts.470.up_proj",
"mtp.layers.0.mlp.experts.471.down_proj",
"mtp.layers.0.mlp.experts.471.gate_proj",
"mtp.layers.0.mlp.experts.471.up_proj",
"mtp.layers.0.mlp.experts.472.down_proj",
"mtp.layers.0.mlp.experts.472.gate_proj",
"mtp.layers.0.mlp.experts.472.up_proj",
"mtp.layers.0.mlp.experts.473.down_proj",
"mtp.layers.0.mlp.experts.473.gate_proj",
"mtp.layers.0.mlp.experts.473.up_proj",
"mtp.layers.0.mlp.experts.474.down_proj",
"mtp.layers.0.mlp.experts.474.gate_proj",
"mtp.layers.0.mlp.experts.474.up_proj",
"mtp.layers.0.mlp.experts.475.down_proj",
"mtp.layers.0.mlp.experts.475.gate_proj",
"mtp.layers.0.mlp.experts.475.up_proj",
"mtp.layers.0.mlp.experts.476.down_proj",
"mtp.layers.0.mlp.experts.476.gate_proj",
"mtp.layers.0.mlp.experts.476.up_proj",
"mtp.layers.0.mlp.experts.477.down_proj",
"mtp.layers.0.mlp.experts.477.gate_proj",
"mtp.layers.0.mlp.experts.477.up_proj",
"mtp.layers.0.mlp.experts.478.down_proj",
"mtp.layers.0.mlp.experts.478.gate_proj",
"mtp.layers.0.mlp.experts.478.up_proj",
"mtp.layers.0.mlp.experts.479.down_proj",
"mtp.layers.0.mlp.experts.479.gate_proj",
"mtp.layers.0.mlp.experts.479.up_proj",
"mtp.layers.0.mlp.experts.48.down_proj",
"mtp.layers.0.mlp.experts.48.gate_proj",
"mtp.layers.0.mlp.experts.48.up_proj",
"mtp.layers.0.mlp.experts.480.down_proj",
"mtp.layers.0.mlp.experts.480.gate_proj",
"mtp.layers.0.mlp.experts.480.up_proj",
"mtp.layers.0.mlp.experts.481.down_proj",
"mtp.layers.0.mlp.experts.481.gate_proj",
"mtp.layers.0.mlp.experts.481.up_proj",
"mtp.layers.0.mlp.experts.482.down_proj",
"mtp.layers.0.mlp.experts.482.gate_proj",
"mtp.layers.0.mlp.experts.482.up_proj",
"mtp.layers.0.mlp.experts.483.down_proj",
"mtp.layers.0.mlp.experts.483.gate_proj",
"mtp.layers.0.mlp.experts.483.up_proj",
"mtp.layers.0.mlp.experts.484.down_proj",
"mtp.layers.0.mlp.experts.484.gate_proj",
"mtp.layers.0.mlp.experts.484.up_proj",
"mtp.layers.0.mlp.experts.485.down_proj",
"mtp.layers.0.mlp.experts.485.gate_proj",
"mtp.layers.0.mlp.experts.485.up_proj",
"mtp.layers.0.mlp.experts.486.down_proj",
"mtp.layers.0.mlp.experts.486.gate_proj",
"mtp.layers.0.mlp.experts.486.up_proj",
"mtp.layers.0.mlp.experts.487.down_proj",
"mtp.layers.0.mlp.experts.487.gate_proj",
"mtp.layers.0.mlp.experts.487.up_proj",
"mtp.layers.0.mlp.experts.488.down_proj",
"mtp.layers.0.mlp.experts.488.gate_proj",
"mtp.layers.0.mlp.experts.488.up_proj",
"mtp.layers.0.mlp.experts.489.down_proj",
"mtp.layers.0.mlp.experts.489.gate_proj",
"mtp.layers.0.mlp.experts.489.up_proj",
"mtp.layers.0.mlp.experts.49.down_proj",
"mtp.layers.0.mlp.experts.49.gate_proj",
"mtp.layers.0.mlp.experts.49.up_proj",
"mtp.layers.0.mlp.experts.490.down_proj",
"mtp.layers.0.mlp.experts.490.gate_proj",
"mtp.layers.0.mlp.experts.490.up_proj",
"mtp.layers.0.mlp.experts.491.down_proj",
"mtp.layers.0.mlp.experts.491.gate_proj",
"mtp.layers.0.mlp.experts.491.up_proj",
"mtp.layers.0.mlp.experts.492.down_proj",
"mtp.layers.0.mlp.experts.492.gate_proj",
"mtp.layers.0.mlp.experts.492.up_proj",
"mtp.layers.0.mlp.experts.493.down_proj",
"mtp.layers.0.mlp.experts.493.gate_proj",
"mtp.layers.0.mlp.experts.493.up_proj",
"mtp.layers.0.mlp.experts.494.down_proj",
"mtp.layers.0.mlp.experts.494.gate_proj",
"mtp.layers.0.mlp.experts.494.up_proj",
"mtp.layers.0.mlp.experts.495.down_proj",
"mtp.layers.0.mlp.experts.495.gate_proj",
"mtp.layers.0.mlp.experts.495.up_proj",
"mtp.layers.0.mlp.experts.496.down_proj",
"mtp.layers.0.mlp.experts.496.gate_proj",
"mtp.layers.0.mlp.experts.496.up_proj",
"mtp.layers.0.mlp.experts.497.down_proj",
"mtp.layers.0.mlp.experts.497.gate_proj",
"mtp.layers.0.mlp.experts.497.up_proj",
"mtp.layers.0.mlp.experts.498.down_proj",
"mtp.layers.0.mlp.experts.498.gate_proj",
"mtp.layers.0.mlp.experts.498.up_proj",
"mtp.layers.0.mlp.experts.499.down_proj",
"mtp.layers.0.mlp.experts.499.gate_proj",
"mtp.layers.0.mlp.experts.499.up_proj",
"mtp.layers.0.mlp.experts.5.down_proj",
"mtp.layers.0.mlp.experts.5.gate_proj",
"mtp.layers.0.mlp.experts.5.up_proj",
"mtp.layers.0.mlp.experts.50.down_proj",
"mtp.layers.0.mlp.experts.50.gate_proj",
"mtp.layers.0.mlp.experts.50.up_proj",
"mtp.layers.0.mlp.experts.500.down_proj",
"mtp.layers.0.mlp.experts.500.gate_proj",
"mtp.layers.0.mlp.experts.500.up_proj",
"mtp.layers.0.mlp.experts.501.down_proj",
"mtp.layers.0.mlp.experts.501.gate_proj",
"mtp.layers.0.mlp.experts.501.up_proj",
"mtp.layers.0.mlp.experts.502.down_proj",
"mtp.layers.0.mlp.experts.502.gate_proj",
"mtp.layers.0.mlp.experts.502.up_proj",
"mtp.layers.0.mlp.experts.503.down_proj",
"mtp.layers.0.mlp.experts.503.gate_proj",
"mtp.layers.0.mlp.experts.503.up_proj",
"mtp.layers.0.mlp.experts.504.down_proj",
"mtp.layers.0.mlp.experts.504.gate_proj",
"mtp.layers.0.mlp.experts.504.up_proj",
"mtp.layers.0.mlp.experts.505.down_proj",
"mtp.layers.0.mlp.experts.505.gate_proj",
"mtp.layers.0.mlp.experts.505.up_proj",
"mtp.layers.0.mlp.experts.506.down_proj",
"mtp.layers.0.mlp.experts.506.gate_proj",
"mtp.layers.0.mlp.experts.506.up_proj",
"mtp.layers.0.mlp.experts.507.down_proj",
"mtp.layers.0.mlp.experts.507.gate_proj",
"mtp.layers.0.mlp.experts.507.up_proj",
"mtp.layers.0.mlp.experts.508.down_proj",
"mtp.layers.0.mlp.experts.508.gate_proj",
"mtp.layers.0.mlp.experts.508.up_proj",
"mtp.layers.0.mlp.experts.509.down_proj",
"mtp.layers.0.mlp.experts.509.gate_proj",
"mtp.layers.0.mlp.experts.509.up_proj",
"mtp.layers.0.mlp.experts.51.down_proj",
"mtp.layers.0.mlp.experts.51.gate_proj",
"mtp.layers.0.mlp.experts.51.up_proj",
"mtp.layers.0.mlp.experts.510.down_proj",
"mtp.layers.0.mlp.experts.510.gate_proj",
"mtp.layers.0.mlp.experts.510.up_proj",
"mtp.layers.0.mlp.experts.511.down_proj",
"mtp.layers.0.mlp.experts.511.gate_proj",
"mtp.layers.0.mlp.experts.511.up_proj",
"mtp.layers.0.mlp.experts.52.down_proj",
"mtp.layers.0.mlp.experts.52.gate_proj",
"mtp.layers.0.mlp.experts.52.up_proj",
"mtp.layers.0.mlp.experts.53.down_proj",
"mtp.layers.0.mlp.experts.53.gate_proj",
"mtp.layers.0.mlp.experts.53.up_proj",
"mtp.layers.0.mlp.experts.54.down_proj",
"mtp.layers.0.mlp.experts.54.gate_proj",
"mtp.layers.0.mlp.experts.54.up_proj",
"mtp.layers.0.mlp.experts.55.down_proj",
"mtp.layers.0.mlp.experts.55.gate_proj",
"mtp.layers.0.mlp.experts.55.up_proj",
"mtp.layers.0.mlp.experts.56.down_proj",
"mtp.layers.0.mlp.experts.56.gate_proj",
"mtp.layers.0.mlp.experts.56.up_proj",
"mtp.layers.0.mlp.experts.57.down_proj",
"mtp.layers.0.mlp.experts.57.gate_proj",
"mtp.layers.0.mlp.experts.57.up_proj",
"mtp.layers.0.mlp.experts.58.down_proj",
"mtp.layers.0.mlp.experts.58.gate_proj",
"mtp.layers.0.mlp.experts.58.up_proj",
"mtp.layers.0.mlp.experts.59.down_proj",
"mtp.layers.0.mlp.experts.59.gate_proj",
"mtp.layers.0.mlp.experts.59.up_proj",
"mtp.layers.0.mlp.experts.6.down_proj",
"mtp.layers.0.mlp.experts.6.gate_proj",
"mtp.layers.0.mlp.experts.6.up_proj",
"mtp.layers.0.mlp.experts.60.down_proj",
"mtp.layers.0.mlp.experts.60.gate_proj",
"mtp.layers.0.mlp.experts.60.up_proj",
"mtp.layers.0.mlp.experts.61.down_proj",
"mtp.layers.0.mlp.experts.61.gate_proj",
"mtp.layers.0.mlp.experts.61.up_proj",
"mtp.layers.0.mlp.experts.62.down_proj",
"mtp.layers.0.mlp.experts.62.gate_proj",
"mtp.layers.0.mlp.experts.62.up_proj",
"mtp.layers.0.mlp.experts.63.down_proj",
"mtp.layers.0.mlp.experts.63.gate_proj",
"mtp.layers.0.mlp.experts.63.up_proj",
"mtp.layers.0.mlp.experts.64.down_proj",
"mtp.layers.0.mlp.experts.64.gate_proj",
"mtp.layers.0.mlp.experts.64.up_proj",
"mtp.layers.0.mlp.experts.65.down_proj",
"mtp.layers.0.mlp.experts.65.gate_proj",
"mtp.layers.0.mlp.experts.65.up_proj",
"mtp.layers.0.mlp.experts.66.down_proj",
"mtp.layers.0.mlp.experts.66.gate_proj",
"mtp.layers.0.mlp.experts.66.up_proj",
"mtp.layers.0.mlp.experts.67.down_proj",
"mtp.layers.0.mlp.experts.67.gate_proj",
"mtp.layers.0.mlp.experts.67.up_proj",
"mtp.layers.0.mlp.experts.68.down_proj",
"mtp.layers.0.mlp.experts.68.gate_proj",
"mtp.layers.0.mlp.experts.68.up_proj",
"mtp.layers.0.mlp.experts.69.down_proj",
"mtp.layers.0.mlp.experts.69.gate_proj",
"mtp.layers.0.mlp.experts.69.up_proj",
"mtp.layers.0.mlp.experts.7.down_proj",
"mtp.layers.0.mlp.experts.7.gate_proj",
"mtp.layers.0.mlp.experts.7.up_proj",
"mtp.layers.0.mlp.experts.70.down_proj",
"mtp.layers.0.mlp.experts.70.gate_proj",
"mtp.layers.0.mlp.experts.70.up_proj",
"mtp.layers.0.mlp.experts.71.down_proj",
"mtp.layers.0.mlp.experts.71.gate_proj",
"mtp.layers.0.mlp.experts.71.up_proj",
"mtp.layers.0.mlp.experts.72.down_proj",
"mtp.layers.0.mlp.experts.72.gate_proj",
"mtp.layers.0.mlp.experts.72.up_proj",
"mtp.layers.0.mlp.experts.73.down_proj",
"mtp.layers.0.mlp.experts.73.gate_proj",
"mtp.layers.0.mlp.experts.73.up_proj",
"mtp.layers.0.mlp.experts.74.down_proj",
"mtp.layers.0.mlp.experts.74.gate_proj",
"mtp.layers.0.mlp.experts.74.up_proj",
"mtp.layers.0.mlp.experts.75.down_proj",
"mtp.layers.0.mlp.experts.75.gate_proj",
"mtp.layers.0.mlp.experts.75.up_proj",
"mtp.layers.0.mlp.experts.76.down_proj",
"mtp.layers.0.mlp.experts.76.gate_proj",
"mtp.layers.0.mlp.experts.76.up_proj",
"mtp.layers.0.mlp.experts.77.down_proj",
"mtp.layers.0.mlp.experts.77.gate_proj",
"mtp.layers.0.mlp.experts.77.up_proj",
"mtp.layers.0.mlp.experts.78.down_proj",
"mtp.layers.0.mlp.experts.78.gate_proj",
"mtp.layers.0.mlp.experts.78.up_proj",
"mtp.layers.0.mlp.experts.79.down_proj",
"mtp.layers.0.mlp.experts.79.gate_proj",
"mtp.layers.0.mlp.experts.79.up_proj",
"mtp.layers.0.mlp.experts.8.down_proj",
"mtp.layers.0.mlp.experts.8.gate_proj",
"mtp.layers.0.mlp.experts.8.up_proj",
"mtp.layers.0.mlp.experts.80.down_proj",
"mtp.layers.0.mlp.experts.80.gate_proj",
"mtp.layers.0.mlp.experts.80.up_proj",
"mtp.layers.0.mlp.experts.81.down_proj",
"mtp.layers.0.mlp.experts.81.gate_proj",
"mtp.layers.0.mlp.experts.81.up_proj",
"mtp.layers.0.mlp.experts.82.down_proj",
"mtp.layers.0.mlp.experts.82.gate_proj",
"mtp.layers.0.mlp.experts.82.up_proj",
"mtp.layers.0.mlp.experts.83.down_proj",
"mtp.layers.0.mlp.experts.83.gate_proj",
"mtp.layers.0.mlp.experts.83.up_proj",
"mtp.layers.0.mlp.experts.84.down_proj",
"mtp.layers.0.mlp.experts.84.gate_proj",
"mtp.layers.0.mlp.experts.84.up_proj",
"mtp.layers.0.mlp.experts.85.down_proj",
"mtp.layers.0.mlp.experts.85.gate_proj",
"mtp.layers.0.mlp.experts.85.up_proj",
"mtp.layers.0.mlp.experts.86.down_proj",
"mtp.layers.0.mlp.experts.86.gate_proj",
"mtp.layers.0.mlp.experts.86.up_proj",
"mtp.layers.0.mlp.experts.87.down_proj",
"mtp.layers.0.mlp.experts.87.gate_proj",
"mtp.layers.0.mlp.experts.87.up_proj",
"mtp.layers.0.mlp.experts.88.down_proj",
"mtp.layers.0.mlp.experts.88.gate_proj",
"mtp.layers.0.mlp.experts.88.up_proj",
"mtp.layers.0.mlp.experts.89.down_proj",
"mtp.layers.0.mlp.experts.89.gate_proj",
"mtp.layers.0.mlp.experts.89.up_proj",
"mtp.layers.0.mlp.experts.9.down_proj",
"mtp.layers.0.mlp.experts.9.gate_proj",
"mtp.layers.0.mlp.experts.9.up_proj",
"mtp.layers.0.mlp.experts.90.down_proj",
"mtp.layers.0.mlp.experts.90.gate_proj",
"mtp.layers.0.mlp.experts.90.up_proj",
"mtp.layers.0.mlp.experts.91.down_proj",
"mtp.layers.0.mlp.experts.91.gate_proj",
"mtp.layers.0.mlp.experts.91.up_proj",
"mtp.layers.0.mlp.experts.92.down_proj",
"mtp.layers.0.mlp.experts.92.gate_proj",
"mtp.layers.0.mlp.experts.92.up_proj",
"mtp.layers.0.mlp.experts.93.down_proj",
"mtp.layers.0.mlp.experts.93.gate_proj",
"mtp.layers.0.mlp.experts.93.up_proj",
"mtp.layers.0.mlp.experts.94.down_proj",
"mtp.layers.0.mlp.experts.94.gate_proj",
"mtp.layers.0.mlp.experts.94.up_proj",
"mtp.layers.0.mlp.experts.95.down_proj",
"mtp.layers.0.mlp.experts.95.gate_proj",
"mtp.layers.0.mlp.experts.95.up_proj",
"mtp.layers.0.mlp.experts.96.down_proj",
"mtp.layers.0.mlp.experts.96.gate_proj",
"mtp.layers.0.mlp.experts.96.up_proj",
"mtp.layers.0.mlp.experts.97.down_proj",
"mtp.layers.0.mlp.experts.97.gate_proj",
"mtp.layers.0.mlp.experts.97.up_proj",
"mtp.layers.0.mlp.experts.98.down_proj",
"mtp.layers.0.mlp.experts.98.gate_proj",
"mtp.layers.0.mlp.experts.98.up_proj",
"mtp.layers.0.mlp.experts.99.down_proj",
"mtp.layers.0.mlp.experts.99.gate_proj",
"mtp.layers.0.mlp.experts.99.up_proj",
"mtp.layers.0.mlp.gate",
"mtp.layers.0.mlp.shared_expert.down_proj",
"mtp.layers.0.mlp.shared_expert.gate_proj",
"mtp.layers.0.mlp.shared_expert.up_proj",
"mtp.layers.0.mlp.shared_expert_gate",
"mtp.layers.0.post_attention_layernorm",
"mtp.layers.0.self_attn.k_norm",
"mtp.layers.0.self_attn.k_proj",
"mtp.layers.0.self_attn.o_proj",
"mtp.layers.0.self_attn.q_norm",
"mtp.layers.0.self_attn.q_proj",
"mtp.layers.0.self_attn.v_proj",
"mtp.norm",
"mtp.pre_fc_norm_embedding",
"mtp.pre_fc_norm_hidden"
],
"kv_cache_scheme": null,
"quant_method": "compressed-tensors",
"quantization_status": "compressed"
}
} |