diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/charsetprober.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/charsetprober.py
new file mode 100644
index 0000000000000000000000000000000000000000..a103ca11356606402c03b320a4fcdb8635051623
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/charsetprober.py
@@ -0,0 +1,147 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Mark Pilgrim - port to Python
+# Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301 USA
+######################### END LICENSE BLOCK #########################
+
+import logging
+import re
+from typing import Optional, Union
+
+from .enums import LanguageFilter, ProbingState
+
+INTERNATIONAL_WORDS_PATTERN = re.compile(
+ b"[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?"
+)
+
+
+class CharSetProber:
+
+ SHORTCUT_THRESHOLD = 0.95
+
+ def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+ self._state = ProbingState.DETECTING
+ self.active = True
+ self.lang_filter = lang_filter
+ self.logger = logging.getLogger(__name__)
+
+ def reset(self) -> None:
+ self._state = ProbingState.DETECTING
+
+ @property
+ def charset_name(self) -> Optional[str]:
+ return None
+
+ @property
+ def language(self) -> Optional[str]:
+ raise NotImplementedError
+
+ def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+ raise NotImplementedError
+
+ @property
+ def state(self) -> ProbingState:
+ return self._state
+
+ def get_confidence(self) -> float:
+ return 0.0
+
+ @staticmethod
+ def filter_high_byte_only(buf: Union[bytes, bytearray]) -> bytes:
+ buf = re.sub(b"([\x00-\x7F])+", b" ", buf)
+ return buf
+
+ @staticmethod
+ def filter_international_words(buf: Union[bytes, bytearray]) -> bytearray:
+ """
+ We define three types of bytes:
+ alphabet: english alphabets [a-zA-Z]
+ international: international characters [\x80-\xFF]
+ marker: everything else [^a-zA-Z\x80-\xFF]
+ The input buffer can be thought to contain a series of words delimited
+ by markers. This function works to filter all words that contain at
+ least one international character. All contiguous sequences of markers
+ are replaced by a single space ascii character.
+ This filter applies to all scripts which do not use English characters.
+ """
+ filtered = bytearray()
+
+ # This regex expression filters out only words that have at-least one
+ # international character. The word may include one marker character at
+ # the end.
+ words = INTERNATIONAL_WORDS_PATTERN.findall(buf)
+
+ for word in words:
+ filtered.extend(word[:-1])
+
+ # If the last character in the word is a marker, replace it with a
+ # space as markers shouldn't affect our analysis (they are used
+ # similarly across all languages and may thus have similar
+ # frequencies).
+ last_char = word[-1:]
+ if not last_char.isalpha() and last_char < b"\x80":
+ last_char = b" "
+ filtered.extend(last_char)
+
+ return filtered
+
+ @staticmethod
+ def remove_xml_tags(buf: Union[bytes, bytearray]) -> bytes:
+ """
+ Returns a copy of ``buf`` that retains only the sequences of English
+ alphabet and high byte characters that are not between <> characters.
+ This filter can be applied to all scripts which contain both English
+ characters and extended ASCII characters, but is currently only used by
+ ``Latin1Prober``.
+ """
+ filtered = bytearray()
+ in_tag = False
+ prev = 0
+ buf = memoryview(buf).cast("c")
+
+ for curr, buf_char in enumerate(buf):
+ # Check if we're coming out of or entering an XML tag
+
+ # https://github.com/python/typeshed/issues/8182
+ if buf_char == b">": # type: ignore[comparison-overlap]
+ prev = curr + 1
+ in_tag = False
+ # https://github.com/python/typeshed/issues/8182
+ elif buf_char == b"<": # type: ignore[comparison-overlap]
+ if curr > prev and not in_tag:
+ # Keep everything after last non-extended-ASCII,
+ # non-alphabetic character
+ filtered.extend(buf[prev:curr])
+ # Output a space to delimit stretch we kept
+ filtered.extend(b" ")
+ in_tag = True
+
+ # If we're not in a tag...
+ if not in_tag:
+ # Keep everything after last non-extended-ASCII, non-alphabetic
+ # character
+ filtered.extend(buf[prev:])
+
+ return filtered
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/euckrprober.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/euckrprober.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fc5de0462cd9a09472cece4087cafe699da4fa7
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/euckrprober.py
@@ -0,0 +1,47 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301 USA
+######################### END LICENSE BLOCK #########################
+
+from .chardistribution import EUCKRDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import EUCKR_SM_MODEL
+
+
+class EUCKRProber(MultiByteCharSetProber):
+ def __init__(self) -> None:
+ super().__init__()
+ self.coding_sm = CodingStateMachine(EUCKR_SM_MODEL)
+ self.distribution_analyzer = EUCKRDistributionAnalysis()
+ self.reset()
+
+ @property
+ def charset_name(self) -> str:
+ return "EUC-KR"
+
+ @property
+ def language(self) -> str:
+ return "Korean"
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/gb2312freq.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/gb2312freq.py
new file mode 100644
index 0000000000000000000000000000000000000000..b32bfc74213d93d434f1f3a47cb5d7d0bf4863d3
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35/lib/python3.12/site-packages/pip/_vendor/chardet/gb2312freq.py
@@ -0,0 +1,284 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301 USA
+######################### END LICENSE BLOCK #########################
+
+# GB2312 most frequently used character table
+#
+# Char to FreqOrder table , from hz6763
+
+# 512 --> 0.79 -- 0.79
+# 1024 --> 0.92 -- 0.13
+# 2048 --> 0.98 -- 0.06
+# 6768 --> 1.00 -- 0.02
+#
+# Ideal Distribution Ratio = 0.79135/(1-0.79135) = 3.79
+# Random Distribution Ration = 512 / (3755 - 512) = 0.157
+#
+# Typical Distribution Ratio about 25% of Ideal one, still much higher that RDR
+
+GB2312_TYPICAL_DISTRIBUTION_RATIO = 0.9
+
+GB2312_TABLE_SIZE = 3760
+
+# fmt: off
+GB2312_CHAR_TO_FREQ_ORDER = (
+1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205,
+2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842,
+2204, 869,4207, 970,2678,5626,2944,2956,1479,4048, 514,3595, 588,1346,2820,3409,
+ 249,4088,1746,1873,2047,1774, 581,1813, 358,1174,3590,1014,1561,4844,2245, 670,
+1636,3112, 889,1286, 953, 556,2327,3060,1290,3141, 613, 185,3477,1367, 850,3820,
+1715,2428,2642,2303,2732,3041,2562,2648,3566,3946,1349, 388,3098,2091,1360,3585,
+ 152,1687,1539, 738,1559, 59,1232,2925,2267,1388,1249,1741,1679,2960, 151,1566,
+1125,1352,4271, 924,4296, 385,3166,4459, 310,1245,2850, 70,3285,2729,3534,3575,
+2398,3298,3466,1960,2265, 217,3647, 864,1909,2084,4401,2773,1010,3269,5152, 853,
+3051,3121,1244,4251,1895, 364,1499,1540,2313,1180,3655,2268, 562, 715,2417,3061,
+ 544, 336,3768,2380,1752,4075, 950, 280,2425,4382, 183,2759,3272, 333,4297,2155,
+1688,2356,1444,1039,4540, 736,1177,3349,2443,2368,2144,2225, 565, 196,1482,3406,
+ 927,1335,4147, 692, 878,1311,1653,3911,3622,1378,4200,1840,2969,3149,2126,1816,
+2534,1546,2393,2760, 737,2494, 13, 447, 245,2747, 38,2765,2129,2589,1079, 606,
+ 360, 471,3755,2890, 404, 848, 699,1785,1236, 370,2221,1023,3746,2074,2026,2023,
+2388,1581,2119, 812,1141,3091,2536,1519, 804,2053, 406,1596,1090, 784, 548,4414,
+1806,2264,2936,1100, 343,4114,5096, 622,3358, 743,3668,1510,1626,5020,3567,2513,
+3195,4115,5627,2489,2991, 24,2065,2697,1087,2719, 48,1634, 315, 68, 985,2052,
+ 198,2239,1347,1107,1439, 597,2366,2172, 871,3307, 919,2487,2790,1867, 236,2570,
+1413,3794, 906,3365,3381,1701,1982,1818,1524,2924,1205, 616,2586,2072,2004, 575,
+ 253,3099, 32,1365,1182, 197,1714,2454,1201, 554,3388,3224,2748, 756,2587, 250,
+2567,1507,1517,3529,1922,2761,2337,3416,1961,1677,2452,2238,3153, 615, 911,1506,
+1474,2495,1265,1906,2749,3756,3280,2161, 898,2714,1759,3450,2243,2444, 563, 26,
+3286,2266,3769,3344,2707,3677, 611,1402, 531,1028,2871,4548,1375, 261,2948, 835,
+1190,4134, 353, 840,2684,1900,3082,1435,2109,1207,1674, 329,1872,2781,4055,2686,
+2104, 608,3318,2423,2957,2768,1108,3739,3512,3271,3985,2203,1771,3520,1418,2054,
+1681,1153, 225,1627,2929, 162,2050,2511,3687,1954, 124,1859,2431,1684,3032,2894,
+ 585,4805,3969,2869,2704,2088,2032,2095,3656,2635,4362,2209, 256, 518,2042,2105,
+3777,3657, 643,2298,1148,1779, 190, 989,3544, 414, 11,2135,2063,2979,1471, 403,
+3678, 126, 770,1563, 671,2499,3216,2877, 600,1179, 307,2805,4937,1268,1297,2694,
+ 252,4032,1448,1494,1331,1394, 127,2256, 222,1647,1035,1481,3056,1915,1048, 873,
+3651, 210, 33,1608,2516, 200,1520, 415, 102, 0,3389,1287, 817, 91,3299,2940,
+ 836,1814, 549,2197,1396,1669,2987,3582,2297,2848,4528,1070, 687, 20,1819, 121,
+1552,1364,1461,1968,2617,3540,2824,2083, 177, 948,4938,2291, 110,4549,2066, 648,
+3359,1755,2110,2114,4642,4845,1693,3937,3308,1257,1869,2123, 208,1804,3159,2992,
+2531,2549,3361,2418,1350,2347,2800,2568,1291,2036,2680, 72, 842,1990, 212,1233,
+1154,1586, 75,2027,3410,4900,1823,1337,2710,2676, 728,2810,1522,3026,4995, 157,
+ 755,1050,4022, 710, 785,1936,2194,2085,1406,2777,2400, 150,1250,4049,1206, 807,
+1910, 534, 529,3309,1721,1660, 274, 39,2827, 661,2670,1578, 925,3248,3815,1094,
+4278,4901,4252, 41,1150,3747,2572,2227,4501,3658,4902,3813,3357,3617,2884,2258,
+ 887, 538,4187,3199,1294,2439,3042,2329,2343,2497,1255, 107, 543,1527, 521,3478,
+3568, 194,5062, 15, 961,3870,1241,1192,2664, 66,5215,3260,2111,1295,1127,2152,
+3805,4135, 901,1164,1976, 398,1278, 530,1460, 748, 904,1054,1966,1426, 53,2909,
+ 509, 523,2279,1534, 536,1019, 239,1685, 460,2353, 673,1065,2401,3600,4298,2272,
+1272,2363, 284,1753,3679,4064,1695, 81, 815,2677,2757,2731,1386, 859, 500,4221,
+2190,2566, 757,1006,2519,2068,1166,1455, 337,2654,3203,1863,1682,1914,3025,1252,
+1409,1366, 847, 714,2834,2038,3209, 964,2970,1901, 885,2553,1078,1756,3049, 301,
+1572,3326, 688,2130,1996,2429,1805,1648,2930,3421,2750,3652,3088, 262,1158,1254,
+ 389,1641,1812, 526,1719, 923,2073,1073,1902, 468, 489,4625,1140, 857,2375,3070,
+3319,2863, 380, 116,1328,2693,1161,2244, 273,1212,1884,2769,3011,1775,1142, 461,
+3066,1200,2147,2212, 790, 702,2695,4222,1601,1058, 434,2338,5153,3640, 67,2360,
+4099,2502, 618,3472,1329, 416,1132, 830,2782,1807,2653,3211,3510,1662, 192,2124,
+ 296,3979,1739,1611,3684, 23, 118, 324, 446,1239,1225, 293,2520,3814,3795,2535,
+3116, 17,1074, 467,2692,2201, 387,2922, 45,1326,3055,1645,3659,2817, 958, 243,
+1903,2320,1339,2825,1784,3289, 356, 576, 865,2315,2381,3377,3916,1088,3122,1713,
+1655, 935, 628,4689,1034,1327, 441, 800, 720, 894,1979,2183,1528,5289,2702,1071,
+4046,3572,2399,1571,3281, 79, 761,1103, 327, 134, 758,1899,1371,1615, 879, 442,
+ 215,2605,2579, 173,2048,2485,1057,2975,3317,1097,2253,3801,4263,1403,1650,2946,
+ 814,4968,3487,1548,2644,1567,1285, 2, 295,2636, 97, 946,3576, 832, 141,4257,
+3273, 760,3821,3521,3156,2607, 949,1024,1733,1516,1803,1920,2125,2283,2665,3180,
+1501,2064,3560,2171,1592, 803,3518,1416, 732,3897,4258,1363,1362,2458, 119,1427,
+ 602,1525,2608,1605,1639,3175, 694,3064, 10, 465, 76,2000,4846,4208, 444,3781,
+1619,3353,2206,1273,3796, 740,2483, 320,1723,2377,3660,2619,1359,1137,1762,1724,
+2345,2842,1850,1862, 912, 821,1866, 612,2625,1735,2573,3369,1093, 844, 89, 937,
+ 930,1424,3564,2413,2972,1004,3046,3019,2011, 711,3171,1452,4178, 428, 801,1943,
+ 432, 445,2811, 206,4136,1472, 730, 349, 73, 397,2802,2547, 998,1637,1167, 789,
+ 396,3217, 154,1218, 716,1120,1780,2819,4826,1931,3334,3762,2139,1215,2627, 552,
+3664,3628,3232,1405,2383,3111,1356,2652,3577,3320,3101,1703, 640,1045,1370,1246,
+4996, 371,1575,2436,1621,2210, 984,4033,1734,2638, 16,4529, 663,2755,3255,1451,
+3917,2257,1253,1955,2234,1263,2951, 214,1229, 617, 485, 359,1831,1969, 473,2310,
+ 750,2058, 165, 80,2864,2419, 361,4344,2416,2479,1134, 796,3726,1266,2943, 860,
+2715, 938, 390,2734,1313,1384, 248, 202, 877,1064,2854, 522,3907, 279,1602, 297,
+2357, 395,3740, 137,2075, 944,4089,2584,1267,3802, 62,1533,2285, 178, 176, 780,
+2440, 201,3707, 590, 478,1560,4354,2117,1075, 30, 74,4643,4004,1635,1441,2745,
+ 776,2596, 238,1077,1692,1912,2844, 605, 499,1742,3947, 241,3053, 980,1749, 936,
+2640,4511,2582, 515,1543,2162,5322,2892,2993, 890,2148,1924, 665,1827,3581,1032,
+ 968,3163, 339,1044,1896, 270, 583,1791,1720,4367,1194,3488,3669, 43,2523,1657,
+ 163,2167, 290,1209,1622,3378, 550, 634,2508,2510, 695,2634,2384,2512,1476,1414,
+ 220,1469,2341,2138,2852,3183,2900,4939,2865,3502,1211,3680, 854,3227,1299,2976,
+3172, 186,2998,1459, 443,1067,3251,1495, 321,1932,3054, 909, 753,1410,1828, 436,
+2441,1119,1587,3164,2186,1258, 227, 231,1425,1890,3200,3942, 247, 959, 725,5254,
+2741, 577,2158,2079, 929, 120, 174, 838,2813, 591,1115, 417,2024, 40,3240,1536,
+1037, 291,4151,2354, 632,1298,2406,2500,3535,1825,1846,3451, 205,1171, 345,4238,
+ 18,1163, 811, 685,2208,1217, 425,1312,1508,1175,4308,2552,1033, 587,1381,3059,
+2984,3482, 340,1316,4023,3972, 792,3176, 519, 777,4690, 918, 933,4130,2981,3741,
+ 90,3360,2911,2200,5184,4550, 609,3079,2030, 272,3379,2736, 363,3881,1130,1447,
+ 286, 779, 357,1169,3350,3137,1630,1220,2687,2391, 747,1277,3688,2618,2682,2601,
+1156,3196,5290,4034,3102,1689,3596,3128, 874, 219,2783, 798, 508,1843,2461, 269,
+1658,1776,1392,1913,2983,3287,2866,2159,2372, 829,4076, 46,4253,2873,1889,1894,
+ 915,1834,1631,2181,2318, 298, 664,2818,3555,2735, 954,3228,3117, 527,3511,2173,
+ 681,2712,3033,2247,2346,3467,1652, 155,2164,3382, 113,1994, 450, 899, 494, 994,
+1237,2958,1875,2336,1926,3727, 545,1577,1550, 633,3473, 204,1305,3072,2410,1956,
+2471, 707,2134, 841,2195,2196,2663,3843,1026,4940, 990,3252,4997, 368,1092, 437,
+3212,3258,1933,1829, 675,2977,2893, 412, 943,3723,4644,3294,3283,2230,2373,5154,
+2389,2241,2661,2323,1404,2524, 593, 787, 677,3008,1275,2059, 438,2709,2609,2240,
+2269,2246,1446, 36,1568,1373,3892,1574,2301,1456,3962, 693,2276,5216,2035,1143,
+2720,1919,1797,1811,2763,4137,2597,1830,1699,1488,1198,2090, 424,1694, 312,3634,
+3390,4179,3335,2252,1214, 561,1059,3243,2295,2561, 975,5155,2321,2751,3772, 472,
+1537,3282,3398,1047,2077,2348,2878,1323,3340,3076, 690,2906, 51, 369, 170,3541,
+1060,2187,2688,3670,2541,1083,1683, 928,3918, 459, 109,4427, 599,3744,4286, 143,
+2101,2730,2490, 82,1588,3036,2121, 281,1860, 477,4035,1238,2812,3020,2716,3312,
+1530,2188,2055,1317, 843, 636,1808,1173,3495, 649, 181,1002, 147,3641,1159,2414,
+3750,2289,2795, 813,3123,2610,1136,4368, 5,3391,4541,2174, 420, 429,1728, 754,
+1228,2115,2219, 347,2223,2733, 735,1518,3003,2355,3134,1764,3948,3329,1888,2424,
+1001,1234,1972,3321,3363,1672,1021,1450,1584, 226, 765, 655,2526,3404,3244,2302,
+3665, 731, 594,2184, 319,1576, 621, 658,2656,4299,2099,3864,1279,2071,2598,2739,
+ 795,3086,3699,3908,1707,2352,2402,1382,3136,2475,1465,4847,3496,3865,1085,3004,
+2591,1084, 213,2287,1963,3565,2250, 822, 793,4574,3187,1772,1789,3050, 595,1484,
+1959,2770,1080,2650, 456, 422,2996, 940,3322,4328,4345,3092,2742, 965,2784, 739,
+4124, 952,1358,2498,2949,2565, 332,2698,2378, 660,2260,2473,4194,3856,2919, 535,
+1260,2651,1208,1428,1300,1949,1303,2942, 433,2455,2450,1251,1946, 614,1269, 641,
+1306,1810,2737,3078,2912, 564,2365,1419,1415,1497,4460,2367,2185,1379,3005,1307,
+3218,2175,1897,3063, 682,1157,4040,4005,1712,1160,1941,1399, 394, 402,2952,1573,
+1151,2986,2404, 862, 299,2033,1489,3006, 346, 171,2886,3401,1726,2932, 168,2533,
+ 47,2507,1030,3735,1145,3370,1395,1318,1579,3609,4560,2857,4116,1457,2529,1965,
+ 504,1036,2690,2988,2405, 745,5871, 849,2397,2056,3081, 863,2359,3857,2096, 99,
+1397,1769,2300,4428,1643,3455,1978,1757,3718,1440, 35,4879,3742,1296,4228,2280,
+ 160,5063,1599,2013, 166, 520,3479,1646,3345,3012, 490,1937,1545,1264,2182,2505,
+1096,1188,1369,1436,2421,1667,2792,2460,1270,2122, 727,3167,2143, 806,1706,1012,
+1800,3037, 960,2218,1882, 805, 139,2456,1139,1521, 851,1052,3093,3089, 342,2039,
+ 744,5097,1468,1502,1585,2087, 223, 939, 326,2140,2577, 892,2481,1623,4077, 982,
+3708, 135,2131, 87,2503,3114,2326,1106, 876,1616, 547,2997,2831,2093,3441,4530,
+4314, 9,3256,4229,4148, 659,1462,1986,1710,2046,2913,2231,4090,4880,5255,3392,
+3274,1368,3689,4645,1477, 705,3384,3635,1068,1529,2941,1458,3782,1509, 100,1656,
+2548, 718,2339, 408,1590,2780,3548,1838,4117,3719,1345,3530, 717,3442,2778,3220,
+2898,1892,4590,3614,3371,2043,1998,1224,3483, 891, 635, 584,2559,3355, 733,1766,
+1729,1172,3789,1891,2307, 781,2982,2271,1957,1580,5773,2633,2005,4195,3097,1535,
+3213,1189,1934,5693,3262, 586,3118,1324,1598, 517,1564,2217,1868,1893,4445,3728,
+2703,3139,1526,1787,1992,3882,2875,1549,1199,1056,2224,1904,2711,5098,4287, 338,
+1993,3129,3489,2689,1809,2815,1997, 957,1855,3898,2550,3275,3057,1105,1319, 627,
+1505,1911,1883,3526, 698,3629,3456,1833,1431, 746, 77,1261,2017,2296,1977,1885,
+ 125,1334,1600, 525,1798,1109,2222,1470,1945, 559,2236,1186,3443,2476,1929,1411,
+2411,3135,1777,3372,2621,1841,1613,3229, 668,1430,1839,2643,2916, 195,1989,2671,
+2358,1387, 629,3205,2293,5256,4439, 123,1310, 888,1879,4300,3021,3605,1003,1162,
+3192,2910,2010, 140,2395,2859, 55,1082,2012,2901, 662, 419,2081,1438, 680,2774,
+4654,3912,1620,1731,1625,5035,4065,2328, 512,1344, 802,5443,2163,2311,2537, 524,
+3399, 98,1155,2103,1918,2606,3925,2816,1393,2465,1504,3773,2177,3963,1478,4346,
+ 180,1113,4655,3461,2028,1698, 833,2696,1235,1322,1594,4408,3623,3013,3225,2040,
+3022, 541,2881, 607,3632,2029,1665,1219, 639,1385,1686,1099,2803,3231,1938,3188,
+2858, 427, 676,2772,1168,2025, 454,3253,2486,3556, 230,1950, 580, 791,1991,1280,
+1086,1974,2034, 630, 257,3338,2788,4903,1017, 86,4790, 966,2789,1995,1696,1131,
+ 259,3095,4188,1308, 179,1463,5257, 289,4107,1248, 42,3413,1725,2288, 896,1947,
+ 774,4474,4254, 604,3430,4264, 392,2514,2588, 452, 237,1408,3018, 988,4531,1970,
+3034,3310, 540,2370,1562,1288,2990, 502,4765,1147, 4,1853,2708, 207, 294,2814,
+4078,2902,2509, 684, 34,3105,3532,2551, 644, 709,2801,2344, 573,1727,3573,3557,
+2021,1081,3100,4315,2100,3681, 199,2263,1837,2385, 146,3484,1195,2776,3949, 997,
+1939,3973,1008,1091,1202,1962,1847,1149,4209,5444,1076, 493, 117,5400,2521, 972,
+1490,2934,1796,4542,2374,1512,2933,2657, 413,2888,1135,2762,2314,2156,1355,2369,
+ 766,2007,2527,2170,3124,2491,2593,2632,4757,2437, 234,3125,3591,1898,1750,1376,
+1942,3468,3138, 570,2127,2145,3276,4131, 962, 132,1445,4196, 19, 941,3624,3480,
+3366,1973,1374,4461,3431,2629, 283,2415,2275, 808,2887,3620,2112,2563,1353,3610,
+ 955,1089,3103,1053, 96, 88,4097, 823,3808,1583, 399, 292,4091,3313, 421,1128,
+ 642,4006, 903,2539,1877,2082, 596, 29,4066,1790, 722,2157, 130, 995,1569, 769,
+1485, 464, 513,2213, 288,1923,1101,2453,4316, 133, 486,2445, 50, 625, 487,2207,
+ 57, 423, 481,2962, 159,3729,1558, 491, 303, 482, 501, 240,2837, 112,3648,2392,
+1783, 362, 8,3433,3422, 610,2793,3277,1390,1284,1654, 21,3823, 734, 367, 623,
+ 193, 287, 374,1009,1483, 816, 476, 313,2255,2340,1262,2150,2899,1146,2581, 782,
+2116,1659,2018,1880, 255,3586,3314,1110,2867,2137,2564, 986,2767,5185,2006, 650,
+ 158, 926, 762, 881,3157,2717,2362,3587, 306,3690,3245,1542,3077,2427,1691,2478,
+2118,2985,3490,2438, 539,2305, 983, 129,1754, 355,4201,2386, 827,2923, 104,1773,
+2838,2771, 411,2905,3919, 376, 767, 122,1114, 828,2422,1817,3506, 266,3460,1007,
+1609,4998, 945,2612,4429,2274, 726,1247,1964,2914,2199,2070,4002,4108, 657,3323,
+1422, 579, 455,2764,4737,1222,2895,1670, 824,1223,1487,2525, 558, 861,3080, 598,
+2659,2515,1967, 752,2583,2376,2214,4180, 977, 704,2464,4999,2622,4109,1210,2961,
+ 819,1541, 142,2284, 44, 418, 457,1126,3730,4347,4626,1644,1876,3671,1864, 302,
+1063,5694, 624, 723,1984,3745,1314,1676,2488,1610,1449,3558,3569,2166,2098, 409,
+1011,2325,3704,2306, 818,1732,1383,1824,1844,3757, 999,2705,3497,1216,1423,2683,
+2426,2954,2501,2726,2229,1475,2554,5064,1971,1794,1666,2014,1343, 783, 724, 191,
+2434,1354,2220,5065,1763,2752,2472,4152, 131, 175,2885,3434, 92,1466,4920,2616,
+3871,3872,3866, 128,1551,1632, 669,1854,3682,4691,4125,1230, 188,2973,3290,1302,
+1213, 560,3266, 917, 763,3909,3249,1760, 868,1958, 764,1782,2097, 145,2277,3774,
+4462, 64,1491,3062, 971,2132,3606,2442, 221,1226,1617, 218, 323,1185,3207,3147,
+ 571, 619,1473,1005,1744,2281, 449,1887,2396,3685, 275, 375,3816,1743,3844,3731,
+ 845,1983,2350,4210,1377, 773, 967,3499,3052,3743,2725,4007,1697,1022,3943,1464,
+3264,2855,2722,1952,1029,2839,2467, 84,4383,2215, 820,1391,2015,2448,3672, 377,
+1948,2168, 797,2545,3536,2578,2645, 94,2874,1678, 405,1259,3071, 771, 546,1315,
+ 470,1243,3083, 895,2468, 981, 969,2037, 846,4181, 653,1276,2928, 14,2594, 557,
+3007,2474, 156, 902,1338,1740,2574, 537,2518, 973,2282,2216,2433,1928, 138,2903,
+1293,2631,1612, 646,3457, 839,2935, 111, 496,2191,2847, 589,3186, 149,3994,2060,
+4031,2641,4067,3145,1870, 37,3597,2136,1025,2051,3009,3383,3549,1121,1016,3261,
+1301, 251,2446,2599,2153, 872,3246, 637, 334,3705, 831, 884, 921,3065,3140,4092,
+2198,1944, 246,2964, 108,2045,1152,1921,2308,1031, 203,3173,4170,1907,3890, 810,
+1401,2003,1690, 506, 647,1242,2828,1761,1649,3208,2249,1589,3709,2931,5156,1708,
+ 498, 666,2613, 834,3817,1231, 184,2851,1124, 883,3197,2261,3710,1765,1553,2658,
+1178,2639,2351, 93,1193, 942,2538,2141,4402, 235,1821, 870,1591,2192,1709,1871,
+3341,1618,4126,2595,2334, 603, 651, 69, 701, 268,2662,3411,2555,1380,1606, 503,
+ 448, 254,2371,2646, 574,1187,2309,1770, 322,2235,1292,1801, 305, 566,1133, 229,
+2067,2057, 706, 167, 483,2002,2672,3295,1820,3561,3067, 316, 378,2746,3452,1112,
+ 136,1981, 507,1651,2917,1117, 285,4591, 182,2580,3522,1304, 335,3303,1835,2504,
+1795,1792,2248, 674,1018,2106,2449,1857,2292,2845, 976,3047,1781,2600,2727,1389,
+1281, 52,3152, 153, 265,3950, 672,3485,3951,4463, 430,1183, 365, 278,2169, 27,
+1407,1336,2304, 209,1340,1730,2202,1852,2403,2883, 979,1737,1062, 631,2829,2542,
+3876,2592, 825,2086,2226,3048,3625, 352,1417,3724, 542, 991, 431,1351,3938,1861,
+2294, 826,1361,2927,3142,3503,1738, 463,2462,2723, 582,1916,1595,2808, 400,3845,
+3891,2868,3621,2254, 58,2492,1123, 910,2160,2614,1372,1603,1196,1072,3385,1700,
+3267,1980, 696, 480,2430, 920, 799,1570,2920,1951,2041,4047,2540,1321,4223,2469,
+3562,2228,1271,2602, 401,2833,3351,2575,5157, 907,2312,1256, 410, 263,3507,1582,
+ 996, 678,1849,2316,1480, 908,3545,2237, 703,2322, 667,1826,2849,1531,2604,2999,
+2407,3146,2151,2630,1786,3711, 469,3542, 497,3899,2409, 858, 837,4446,3393,1274,
+ 786, 620,1845,2001,3311, 484, 308,3367,1204,1815,3691,2332,1532,2557,1842,2020,
+2724,1927,2333,4440, 567, 22,1673,2728,4475,1987,1858,1144,1597, 101,1832,3601,
+ 12, 974,3783,4391, 951,1412, 1,3720, 453,4608,4041, 528,1041,1027,3230,2628,
+1129, 875,1051,3291,1203,2262,1069,2860,2799,2149,2615,3278, 144,1758,3040, 31,
+ 475,1680, 366,2685,3184, 311,1642,4008,2466,5036,1593,1493,2809, 216,1420,1668,
+ 233, 304,2128,3284, 232,1429,1768,1040,2008,3407,2740,2967,2543, 242,2133, 778,
+1565,2022,2620, 505,2189,2756,1098,2273, 372,1614, 708, 553,2846,2094,2278, 169,
+3626,2835,4161, 228,2674,3165, 809,1454,1309, 466,1705,1095, 900,3423, 880,2667,
+3751,5258,2317,3109,2571,4317,2766,1503,1342, 866,4447,1118, 63,2076, 314,1881,
+1348,1061, 172, 978,3515,1747, 532, 511,3970, 6, 601, 905,2699,3300,1751, 276,
+1467,3725,2668, 65,4239,2544,2779,2556,1604, 578,2451,1802, 992,2331,2624,1320,
+3446, 713,1513,1013, 103,2786,2447,1661, 886,1702, 916, 654,3574,2031,1556, 751,
+2178,2821,2179,1498,1538,2176, 271, 914,2251,2080,1325, 638,1953,2937,3877,2432,
+2754, 95,3265,1716, 260,1227,4083, 775, 106,1357,3254, 426,1607, 555,2480, 772,
+1985, 244,2546, 474, 495,1046,2611,1851,2061, 71,2089,1675,2590, 742,3758,2843,
+3222,1433, 267,2180,2576,2826,2233,2092,3913,2435, 956,1745,3075, 856,2113,1116,
+ 451, 3,1988,2896,1398, 993,2463,1878,2049,1341,2718,2721,2870,2108, 712,2904,
+4363,2753,2324, 277,2872,2349,2649, 384, 987, 435, 691,3000, 922, 164,3939, 652,
+1500,1184,4153,2482,3373,2165,4848,2335,3775,3508,3154,2806,2830,1554,2102,1664,
+2530,1434,2408, 893,1547,2623,3447,2832,2242,2532,3169,2856,3223,2078, 49,3770,
+3469, 462, 318, 656,2259,3250,3069, 679,1629,2758, 344,1138,1104,3120,1836,1283,
+3115,2154,1437,4448, 934, 759,1999, 794,2862,1038, 533,2560,1722,2342, 855,2626,
+1197,1663,4476,3127, 85,4240,2528, 25,1111,1181,3673, 407,3470,4561,2679,2713,
+ 768,1925,2841,3986,1544,1165, 932, 373,1240,2146,1930,2673, 721,4766, 354,4333,
+ 391,2963, 187, 61,3364,1442,1102, 330,1940,1767, 341,3809,4118, 393,2496,2062,
+2211, 105, 331, 300, 439, 913,1332, 626, 379,3304,1557, 328, 689,3952, 309,1555,
+ 931, 317,2517,3027, 325, 569, 686,2107,3084, 60,1042,1333,2794, 264,3177,4014,
+1628, 258,3712, 7,4464,1176,1043,1778, 683, 114,1975, 78,1492, 383,1886, 510,
+ 386, 645,5291,2891,2069,3305,4138,3867,2939,2603,2493,1935,1066,1848,3588,1015,
+1282,1289,4609, 697,1453,3044,2666,3611,1856,2412, 54, 719,1330, 568,3778,2459,
+1748, 788, 492, 551,1191,1000, 488,3394,3763, 282,1799, 348,2016,1523,3155,2390,
+1049, 382,2019,1788,1170, 729,2968,3523, 897,3926,2785,2938,3292, 350,2319,3238,
+1718,1717,2655,3453,3143,4465, 161,2889,2980,2009,1421, 56,1908,1640,2387,2232,
+1917,1874,2477,4921, 148, 83,3438, 592,4245,2882,1822,1055, 741, 115,1496,1624,
+ 381,1638,4592,1020, 516,3214, 458, 947,4575,1432, 211,1514,2926,1865,2142, 189,
+ 852,1221,1400,1486, 882,2299,4036, 351, 28,1122, 700,6479,6480,6481,6482,6483, #last 512
+)
+# fmt: on
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b86884b3b7b0819cc58157a6593aa5d53c883b9
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/__init__.py
@@ -0,0 +1,33 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+ from .auto_factory import *
+ from .configuration_auto import *
+ from .feature_extraction_auto import *
+ from .image_processing_auto import *
+ from .modeling_auto import *
+ from .processing_auto import *
+ from .tokenization_auto import *
+ from .video_processing_auto import *
+else:
+ import sys
+
+ _file = globals()["__file__"]
+ sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py
new file mode 100644
index 0000000000000000000000000000000000000000..c6a457de9e4aee50999a31b6d5ebe11ccabe08d0
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py
@@ -0,0 +1,680 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Factory function to build auto-model classes."""
+
+import copy
+import importlib
+import json
+import os
+from collections import OrderedDict
+from collections.abc import Iterator
+from typing import Any, TypeVar
+
+from huggingface_hub import repo_exists
+
+from ...configuration_utils import PreTrainedConfig
+from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
+from ...utils import (
+ CONFIG_NAME,
+ cached_file,
+ copy_func,
+ extract_commit_hash,
+ find_adapter_config_file,
+ is_peft_available,
+ is_torch_available,
+ logging,
+ requires_backends,
+)
+from .configuration_auto import AutoConfig, model_type_to_module_name, replace_list_option_in_docstrings
+
+
+if is_torch_available():
+ from ...generation import GenerationMixin
+
+
+logger = logging.get_logger(__name__)
+
+_T = TypeVar("_T")
+# Tokenizers will depend on packages installed, too much variance and there are no common base or Protocol
+_LazyAutoMappingValue = tuple[type[Any] | None, type[Any] | None]
+
+CLASS_DOCSTRING = """
+ This is a generic model class that will be instantiated as one of the model classes of the library when created
+ with the [`~BaseAutoModelClass.from_pretrained`] class method or the [`~BaseAutoModelClass.from_config`] class
+ method.
+
+ This class cannot be instantiated directly using `__init__()` (throws an error).
+"""
+
+FROM_CONFIG_DOCSTRING = """
+ Instantiates one of the model classes of the library from a configuration.
+
+ Note:
+ Loading a model from its configuration file does **not** load the model weights. It only affects the
+ model's configuration. Use [`~BaseAutoModelClass.from_pretrained`] to load the model weights.
+
+ Args:
+ config ([`PreTrainedConfig`]):
+ The model class to instantiate is selected based on the configuration class:
+
+ List options
+ attn_implementation (`str`, *optional*):
+ The attention implementation to use in the model (if relevant). Can be any of `"eager"` (manual implementation of the attention), `"sdpa"` (using [`F.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)), `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)), or `"flash_attention_3"` (using [Dao-AILab/flash-attention/hopper](https://github.com/Dao-AILab/flash-attention/tree/main/hopper)). By default, if available, SDPA will be used for torch>=2.1.1. The default is otherwise the manual `"eager"` implementation.
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoConfig, BaseAutoModelClass
+
+ >>> # Download configuration from huggingface.co and cache.
+ >>> config = AutoConfig.from_pretrained("checkpoint_placeholder")
+ >>> model = BaseAutoModelClass.from_config(config)
+ ```
+"""
+
+FROM_PRETRAINED_TORCH_DOCSTRING = """
+ Instantiate one of the model classes of the library from a pretrained model.
+
+ The model class to instantiate is selected based on the `model_type` property of the config object (either
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
+ falling back to using pattern matching on `pretrained_model_name_or_path`:
+
+ List options
+
+ The model is set in evaluation mode by default using `model.eval()` (so for instance, dropout modules are
+ deactivated). To train the model, you should first set it back in training mode with `model.train()`
+
+ Args:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ Can be either:
+
+ - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
+ - A path to a *directory* containing model weights saved using
+ [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
+ model_args (additional positional arguments, *optional*):
+ Will be passed along to the underlying model `__init__()` method.
+ config ([`PreTrainedConfig`], *optional*):
+ Configuration for the model to use instead of an automatically loaded configuration. Configuration can
+ be automatically loaded when:
+
+ - The model is a model provided by the library (loaded with the *model id* string of a pretrained
+ model).
+ - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
+ save directory.
+ - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
+ configuration JSON file named *config.json* is found in the directory.
+ state_dict (*dict[str, torch.Tensor]*, *optional*):
+ A state dictionary to use instead of a state dictionary loaded from saved weights file.
+
+ This option can be used if you want to create a model from a pretrained configuration but load your own
+ weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and
+ [`~PreTrainedModel.from_pretrained`] is not a simpler option.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model configuration should be cached if the
+ standard cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force the (re-)download of the model weights and configuration files, overriding the
+ cached versions if they exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ output_loading_info(`bool`, *optional*, defaults to `False`):
+ Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages.
+ local_files_only(`bool`, *optional*, defaults to `False`):
+ Whether or not to only look at local files (e.g., not try downloading the model).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ trust_remote_code (`bool`, *optional*, defaults to `False`):
+ Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
+ code_revision (`str`, *optional*, defaults to `"main"`):
+ The specific revision to use for the code on the Hub, if the code leaves in a different repository than
+ the rest of the model. It can be a branch name, a tag name, or a commit id, since we use a git-based
+ system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier
+ allowed by git.
+ kwargs (additional keyword arguments, *optional*):
+ Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
+ `output_attentions=True`). Behaves differently depending on whether a `config` is provided or
+ automatically loaded:
+
+ - If a configuration is provided with `config`, `**kwargs` will be directly passed to the
+ underlying model's `__init__` method (we assume all relevant updates to the configuration have
+ already been done)
+ - If a configuration is not provided, `kwargs` will be first passed to the configuration class
+ initialization function ([`~PreTrainedConfig.from_pretrained`]). Each key of `kwargs` that
+ corresponds to a configuration attribute will be used to override said attribute with the
+ supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
+ will be passed to the underlying model's `__init__` function.
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoConfig, BaseAutoModelClass
+
+ >>> # Download model and configuration from huggingface.co and cache.
+ >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder")
+
+ >>> # Update configuration during loading
+ >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder", output_attentions=True)
+ >>> model.config.output_attentions
+ True
+ ```
+"""
+
+
+def _get_model_class(config, model_mapping):
+ supported_models = model_mapping[type(config)]
+ if not isinstance(supported_models, (list, tuple)):
+ return supported_models
+
+ name_to_model = {model.__name__: model for model in supported_models}
+ architectures = getattr(config, "architectures", [])
+ for arch in architectures:
+ if arch in name_to_model:
+ return name_to_model[arch]
+
+ # If not architecture is set in the config or match the supported models, the first element of the tuple is the
+ # defaults.
+ return supported_models[0]
+
+
+class _BaseAutoModelClass:
+ # Base class for auto models.
+ _model_mapping = None
+
+ def __init__(self, *args, **kwargs) -> None:
+ raise OSError(
+ f"{self.__class__.__name__} is designed to be instantiated "
+ f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or "
+ f"`{self.__class__.__name__}.from_config(config)` methods."
+ )
+
+ @classmethod
+ def from_config(cls, config, **kwargs):
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
+ has_remote_code = hasattr(config, "auto_map") and cls.__name__ in config.auto_map
+ has_local_code = type(config) in cls._model_mapping
+ explicit_local_code = has_local_code and not _get_model_class(
+ config, cls._model_mapping
+ ).__module__.startswith("transformers.")
+ if has_remote_code:
+ class_ref = config.auto_map[cls.__name__]
+ if "--" in class_ref:
+ upstream_repo = class_ref.split("--")[0]
+ else:
+ upstream_repo = None
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code, config._name_or_path, has_local_code, has_remote_code, upstream_repo=upstream_repo
+ )
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ if "--" in class_ref:
+ repo_id, class_ref = class_ref.split("--")
+ else:
+ repo_id = config.name_or_path
+ model_class = get_class_from_dynamic_module(class_ref, repo_id, **kwargs)
+ # This block handles the case where the user is loading a model with `trust_remote_code=True`
+ # but a library model exists with the same name. We don't want to override the autoclass
+ # mappings in this case, or all future loads of that model will be the remote code model.
+ if not has_local_code:
+ cls.register(config.__class__, model_class, exist_ok=True)
+ model_class.register_for_auto_class(auto_class=cls)
+ _ = kwargs.pop("code_revision", None)
+ model_class = add_generation_mixin_to_remote_model(model_class)
+ return model_class._from_config(config, **kwargs)
+ elif has_local_code:
+ model_class = _get_model_class(config, cls._model_mapping)
+ if model_class.config_class == config.sub_configs.get("text_config", None):
+ # TODO: Validate that copying the parent quantization config to the text sub-config preserves
+ # modules_to_not_convert and skip-module matching when composite-model module prefixes differ.
+ parent_config = config
+ config = config.get_text_config()
+ # Check both `quantization_config` being present and also not null,
+ # as a `config.json` can have `"quantization_config": null` in it
+ parent_quant = getattr(parent_config, "quantization_config", None)
+ if parent_quant is not None:
+ config.quantization_config = parent_quant
+ return model_class._from_config(config, **kwargs)
+
+ raise ValueError(
+ f"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\n"
+ f"Model type should be one of {', '.join(c.__name__ for c in cls._model_mapping)}."
+ )
+
+ @classmethod
+ def _prepare_config_for_auto_class(cls, config: PreTrainedConfig) -> PreTrainedConfig:
+ """Additional autoclass-specific config post-loading manipulation. May be overridden in subclasses."""
+ return config
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike[str], *model_args, **kwargs):
+ config = kwargs.pop("config", None)
+ trust_remote_code = kwargs.get("trust_remote_code")
+ kwargs["_from_auto"] = True
+ hub_kwargs_names = [
+ "cache_dir",
+ "force_download",
+ "local_files_only",
+ "proxies",
+ "revision",
+ "subfolder",
+ "token",
+ ]
+ hub_kwargs = {name: kwargs.pop(name) for name in hub_kwargs_names if name in kwargs}
+ code_revision = kwargs.pop("code_revision", None)
+ commit_hash = kwargs.pop("_commit_hash", None)
+ adapter_kwargs = kwargs.pop("adapter_kwargs", None)
+
+ token = hub_kwargs.pop("token", None)
+
+ if token is not None:
+ hub_kwargs["token"] = token
+
+ if commit_hash is None:
+ if not isinstance(config, PreTrainedConfig):
+ # We make a call to the config file first (which may be absent) to get the commit hash as soon as possible
+ resolved_config_file = cached_file(
+ pretrained_model_name_or_path,
+ CONFIG_NAME,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ **hub_kwargs,
+ )
+ commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
+ else:
+ commit_hash = getattr(config, "_commit_hash", None)
+
+ if is_peft_available():
+ if adapter_kwargs is None:
+ adapter_kwargs = {}
+ adapter_kwargs = adapter_kwargs.copy() # avoid mutating original
+ if token is not None:
+ adapter_kwargs["token"] = token
+
+ maybe_adapter_path = find_adapter_config_file(
+ pretrained_model_name_or_path, _commit_hash=commit_hash, **adapter_kwargs
+ )
+
+ if maybe_adapter_path is not None:
+ with open(maybe_adapter_path, "r", encoding="utf-8") as f:
+ adapter_config = json.load(f)
+
+ adapter_kwargs["_adapter_model_path"] = pretrained_model_name_or_path
+ # Only override the model name/path if the current value doesn't point to a
+ # complete model with an embedded adapter so that local models with embedded
+ # adapters will load from the local base model rather than pull the base
+ # model named in the adapter's config from the hub.
+ if not os.path.exists(pretrained_model_name_or_path) or not os.path.exists(
+ os.path.join(pretrained_model_name_or_path, CONFIG_NAME)
+ ):
+ pretrained_model_name_or_path = adapter_config["base_model_name_or_path"]
+
+ if not isinstance(config, PreTrainedConfig):
+ kwargs_orig = copy.deepcopy(kwargs)
+ # ensure not to pollute the config object with dtype="auto" - since it's
+ # meaningless in the context of the config object - torch.dtype values are acceptable
+ if kwargs.get("torch_dtype") == "auto":
+ _ = kwargs.pop("torch_dtype")
+ if kwargs.get("dtype") == "auto":
+ _ = kwargs.pop("dtype")
+ # to not overwrite the quantization_config if config has a quantization_config
+ if kwargs.get("quantization_config") is not None:
+ _ = kwargs.pop("quantization_config")
+
+ config, kwargs = AutoConfig.from_pretrained(
+ pretrained_model_name_or_path,
+ return_unused_kwargs=True,
+ code_revision=code_revision,
+ _commit_hash=commit_hash,
+ **hub_kwargs,
+ **kwargs,
+ )
+
+ # if torch_dtype=auto was passed here, ensure to pass it on
+ if kwargs_orig.get("torch_dtype", None) == "auto":
+ kwargs["torch_dtype"] = "auto"
+ if kwargs_orig.get("dtype", None) == "auto":
+ kwargs["dtype"] = "auto"
+ if kwargs_orig.get("quantization_config", None) is not None:
+ kwargs["quantization_config"] = kwargs_orig["quantization_config"]
+
+ has_remote_code = hasattr(config, "auto_map") and cls.__name__ in config.auto_map
+ has_local_code = type(config) in cls._model_mapping
+ explicit_local_code = has_local_code and not _get_model_class(
+ config, cls._model_mapping
+ ).__module__.startswith("transformers.")
+ upstream_repo = None
+ if has_remote_code:
+ class_ref = config.auto_map[cls.__name__]
+ if "--" in class_ref:
+ upstream_repo = class_ref.split("--")[0]
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code,
+ pretrained_model_name_or_path,
+ has_local_code,
+ has_remote_code,
+ upstream_repo=upstream_repo,
+ )
+ kwargs["trust_remote_code"] = trust_remote_code
+
+ # Set the adapter kwargs
+ kwargs["adapter_kwargs"] = adapter_kwargs
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ model_class = get_class_from_dynamic_module(
+ class_ref, pretrained_model_name_or_path, code_revision=code_revision, **hub_kwargs, **kwargs
+ )
+ _ = hub_kwargs.pop("code_revision", None)
+ # This block handles the case where the user is loading a model with `trust_remote_code=True`
+ # but a library model exists with the same name. We don't want to override the autoclass
+ # mappings in this case, or all future loads of that model will be the remote code model.
+ if not has_local_code:
+ cls.register(config.__class__, model_class, exist_ok=True)
+ model_class.register_for_auto_class(auto_class=cls)
+ model_class = add_generation_mixin_to_remote_model(model_class)
+ return model_class.from_pretrained(
+ pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
+ )
+ elif has_local_code:
+ model_class = _get_model_class(config, cls._model_mapping)
+ if model_class.config_class == config.sub_configs.get("text_config", None):
+ # TODO: Validate that copying the parent quantization config to the text sub-config preserves
+ # modules_to_not_convert and skip-module matching when composite-model module prefixes differ.
+ parent_config = config
+ config = config.get_text_config()
+ # Check both `quantization_config` being present and also not null,
+ # as a `config.json` can have `"quantization_config": null` in it
+ parent_quant = getattr(parent_config, "quantization_config", None)
+ if parent_quant is not None:
+ config.quantization_config = parent_quant
+ return model_class.from_pretrained(
+ pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
+ )
+ raise ValueError(
+ f"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\n"
+ f"Model type should be one of {', '.join(c.__name__ for c in cls._model_mapping)}."
+ )
+
+ @classmethod
+ def register(cls, config_class, model_class, exist_ok=False) -> None:
+ """
+ Register a new model for this class.
+
+ Args:
+ config_class ([`PreTrainedConfig`]):
+ The configuration corresponding to the model to register.
+ model_class ([`PreTrainedModel`]):
+ The model to register.
+ """
+ if hasattr(model_class, "config_class") and model_class.config_class.__name__ != config_class.__name__:
+ raise ValueError(
+ "The model class you are passing has a `config_class` attribute that is not consistent with the "
+ f"config class you passed (model has {model_class.config_class} and you passed {config_class}. Fix "
+ "one of those so they match!"
+ )
+ cls._model_mapping.register(config_class, model_class, exist_ok=exist_ok)
+
+
+class _BaseAutoBackboneClass(_BaseAutoModelClass):
+ # Base class for auto backbone models.
+ _model_mapping = None
+
+ @classmethod
+ def _load_timm_backbone_from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+ requires_backends(cls, ["vision", "timm"])
+ from ...models.timm_backbone import TimmBackboneConfig
+
+ config = kwargs.pop("config", TimmBackboneConfig())
+
+ if kwargs.get("out_features") is not None:
+ raise ValueError("Cannot specify `out_features` for timm backbones")
+
+ if kwargs.get("output_loading_info", False):
+ raise ValueError("Cannot specify `output_loading_info=True` when loading from timm")
+
+ num_channels = kwargs.pop("num_channels", config.num_channels)
+ features_only = kwargs.pop("features_only", config.features_only)
+ out_indices = kwargs.pop("out_indices", config.out_indices)
+ config = TimmBackboneConfig(
+ backbone=pretrained_model_name_or_path,
+ num_channels=num_channels,
+ features_only=features_only,
+ out_indices=out_indices,
+ )
+ # Always load a pretrained model when `from_pretrained` is called
+ kwargs.pop("use_pretrained_backbone", None)
+ return super().from_config(config, pretrained=True, **kwargs)
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+ kwargs.pop("use_timm_backbone", None)
+ if not repo_exists(pretrained_model_name_or_path):
+ return cls._load_timm_backbone_from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+
+ return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+
+
+def insert_head_doc(docstring, head_doc: str = ""):
+ if len(head_doc) > 0:
+ return docstring.replace(
+ "one of the model classes of the library ",
+ f"one of the model classes of the library (with a {head_doc} head) ",
+ )
+ return docstring.replace(
+ "one of the model classes of the library ", "one of the base model classes of the library "
+ )
+
+
+def auto_class_update(cls, checkpoint_for_example: str = "google-bert/bert-base-cased", head_doc: str = ""):
+ # Create a new class with the right name from the base class
+ model_mapping = cls._model_mapping
+ name = cls.__name__
+ class_docstring = insert_head_doc(CLASS_DOCSTRING, head_doc=head_doc)
+ cls.__doc__ = class_docstring.replace("BaseAutoModelClass", name)
+
+ # Now we need to copy and re-register `from_config` and `from_pretrained` as class methods otherwise we can't
+ # have a specific docstrings for them.
+ from_config = copy_func(_BaseAutoModelClass.from_config)
+ from_config_docstring = insert_head_doc(FROM_CONFIG_DOCSTRING, head_doc=head_doc)
+ from_config_docstring = from_config_docstring.replace("BaseAutoModelClass", name)
+ from_config_docstring = from_config_docstring.replace("checkpoint_placeholder", checkpoint_for_example)
+ from_config.__doc__ = from_config_docstring
+ from_config = replace_list_option_in_docstrings(model_mapping._model_mapping, use_model_types=False)(from_config)
+ cls.from_config = classmethod(from_config)
+
+ from_pretrained_docstring = FROM_PRETRAINED_TORCH_DOCSTRING
+ from_pretrained = copy_func(_BaseAutoModelClass.from_pretrained)
+ from_pretrained_docstring = insert_head_doc(from_pretrained_docstring, head_doc=head_doc)
+ from_pretrained_docstring = from_pretrained_docstring.replace("BaseAutoModelClass", name)
+ from_pretrained_docstring = from_pretrained_docstring.replace("checkpoint_placeholder", checkpoint_for_example)
+ shortcut = checkpoint_for_example.split("/")[-1].split("-")[0]
+ from_pretrained_docstring = from_pretrained_docstring.replace("shortcut_placeholder", shortcut)
+ from_pretrained.__doc__ = from_pretrained_docstring
+ from_pretrained = replace_list_option_in_docstrings(model_mapping._model_mapping)(from_pretrained)
+ cls.from_pretrained = classmethod(from_pretrained)
+ return cls
+
+
+def get_values(model_mapping):
+ result = []
+ for model in model_mapping.values():
+ if isinstance(model, (list, tuple)):
+ result += list(model)
+ else:
+ result.append(model)
+
+ return result
+
+
+def getattribute_from_module(module, attr):
+ if attr is None:
+ return None
+ if isinstance(attr, tuple):
+ return tuple(getattribute_from_module(module, a) for a in attr)
+ if isinstance(attr, dict):
+ return {k: getattribute_from_module(module, v) for k, v in attr.items()}
+ if hasattr(module, attr):
+ return getattr(module, attr)
+ # Some of the mappings have entries model_type -> object of another model type. In that case we try to grab the
+ # object at the top level.
+ transformers_module = importlib.import_module("transformers")
+
+ if module != transformers_module:
+ try:
+ return getattribute_from_module(transformers_module, attr)
+ except ValueError:
+ raise ValueError(f"Could not find {attr} neither in {module} nor in {transformers_module}!")
+ else:
+ raise ValueError(f"Could not find {attr} in {transformers_module}!")
+
+
+def add_generation_mixin_to_remote_model(model_class):
+ """
+ Adds `GenerationMixin` to the inheritance of `model_class`, if `model_class` is a PyTorch model.
+
+ This function is used for backwards compatibility purposes: in v4.45, we've started a deprecation cycle to make
+ `PreTrainedModel` stop inheriting from `GenerationMixin`. Without this function, older models dynamically loaded
+ from the Hub may not have the `generate` method after we remove the inheritance.
+ """
+ # 1. If it is not a PT model (i.e. doesn't inherit Module), do nothing
+ if "torch.nn.modules.module.Module" not in str(model_class.__mro__):
+ return model_class
+
+ # 2. If it already **directly** inherits from GenerationMixin, do nothing
+ if "GenerationMixin" in str(model_class.__bases__):
+ return model_class
+
+ # 3. Prior to v4.45, we could detect whether a model was `generate`-compatible if it had its own `generate` and/or
+ # `prepare_inputs_for_generation` method.
+ has_custom_generate_in_class = hasattr(model_class, "generate") and "GenerationMixin" not in str(
+ getattr(model_class, "generate")
+ )
+ has_custom_prepare_inputs = hasattr(model_class, "prepare_inputs_for_generation") and "GenerationMixin" not in str(
+ getattr(model_class, "prepare_inputs_for_generation")
+ )
+ if has_custom_generate_in_class or has_custom_prepare_inputs:
+ model_class_with_generation_mixin = type(
+ model_class.__name__, (model_class, GenerationMixin), {**model_class.__dict__}
+ )
+ return model_class_with_generation_mixin
+ return model_class
+
+
+class _LazyAutoMapping(OrderedDict[type[PreTrainedConfig], _LazyAutoMappingValue]):
+ """
+ A mapping config to object (model or tokenizer for instance) that will load keys and values when it is accessed.
+
+ Args:
+ - config_mapping: The map model type to config class
+ - model_mapping: The map model type to model (or tokenizer) class
+ """
+
+ def __init__(self, config_mapping, model_mapping) -> None:
+ self._config_mapping = config_mapping
+ self._reverse_config_mapping = {v: k for k, v in config_mapping.items()}
+ self._model_mapping = model_mapping
+ self._model_mapping._model_mapping = self
+ self._extra_content = {}
+ self._modules = {}
+
+ def __len__(self) -> int:
+ common_keys = set(self._config_mapping.keys()).intersection(self._model_mapping.keys())
+ return len(common_keys) + len(self._extra_content)
+
+ def __getitem__(self, key: type[PreTrainedConfig]) -> _LazyAutoMappingValue:
+ if key in self._extra_content:
+ return self._extra_content[key]
+ model_type = self._reverse_config_mapping[key.__name__]
+ if model_type in self._model_mapping:
+ model_name = self._model_mapping[model_type]
+ return self._load_attr_from_module(model_type, model_name)
+
+ # Maybe there was several model types associated with this config.
+ model_types = [k for k, v in self._config_mapping.items() if v == key.__name__]
+ for mtype in model_types:
+ if mtype in self._model_mapping:
+ model_name = self._model_mapping[mtype]
+ return self._load_attr_from_module(mtype, model_name)
+ raise KeyError(key)
+
+ def _load_attr_from_module(self, model_type, attr):
+ module_name = model_type_to_module_name(model_type)
+ if module_name not in self._modules:
+ self._modules[module_name] = importlib.import_module(f".{module_name}", "transformers.models")
+ return getattribute_from_module(self._modules[module_name], attr)
+
+ def keys(self) -> list[type[PreTrainedConfig]]:
+ mapping_keys = [
+ self._load_attr_from_module(key, name)
+ for key, name in self._config_mapping.items()
+ if key in self._model_mapping
+ ]
+ return mapping_keys + list(self._extra_content.keys())
+
+ def get(self, key: type[PreTrainedConfig], default: _T) -> _LazyAutoMappingValue | _T:
+ try:
+ return self.__getitem__(key)
+ except KeyError:
+ return default
+
+ def __bool__(self) -> bool:
+ return bool(self.keys())
+
+ def values(self) -> list[_LazyAutoMappingValue]:
+ mapping_values = [
+ self._load_attr_from_module(key, name)
+ for key, name in self._model_mapping.items()
+ if key in self._config_mapping
+ ]
+ return mapping_values + list(self._extra_content.values())
+
+ def items(self) -> list[tuple[type[PreTrainedConfig], _LazyAutoMappingValue]]:
+ mapping_items = [
+ (
+ self._load_attr_from_module(key, self._config_mapping[key]),
+ self._load_attr_from_module(key, self._model_mapping[key]),
+ )
+ for key in self._model_mapping
+ if key in self._config_mapping
+ ]
+ return mapping_items + list(self._extra_content.items())
+
+ def __iter__(self) -> Iterator[type[PreTrainedConfig]]:
+ return iter(self.keys())
+
+ def __contains__(self, item: type) -> bool:
+ if item in self._extra_content:
+ return True
+ if not hasattr(item, "__name__") or item.__name__ not in self._reverse_config_mapping:
+ return False
+ model_type = self._reverse_config_mapping[item.__name__]
+ return model_type in self._model_mapping
+
+ def register(self, key: type[PreTrainedConfig], value: _LazyAutoMappingValue, exist_ok=False) -> None:
+ """
+ Register a new model in this mapping.
+ """
+ if hasattr(key, "__name__") and key.__name__ in self._reverse_config_mapping:
+ model_type = self._reverse_config_mapping[key.__name__]
+ if model_type in self._model_mapping and not exist_ok:
+ raise ValueError(f"'{key}' is already used by a Transformers model.")
+
+ self._extra_content[key] = value
+
+
+__all__ = ["get_values"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/auto_mappings.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/auto_mappings.py
new file mode 100644
index 0000000000000000000000000000000000000000..26770641fd6cce8f9c128979410284f6dcdc9f69
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/auto_mappings.py
@@ -0,0 +1,1008 @@
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# This file was automatically generated from existing config files and their `model_type`s. Do NOT edit this file
+# manually as any edits will be overwritten by auto-generation of the file. If any change should be done,
+# please add the correct `cls.model_type` in your config class and run `python utils/check_auto.py --fix_and_overwrite`.
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from collections import OrderedDict
+
+
+CONFIG_MAPPING_NAMES = OrderedDict(
+ [
+ ("afmoe", "AfmoeConfig"),
+ ("aimv2", "Aimv2Config"),
+ ("aimv2_text_model", "Aimv2TextConfig"),
+ ("aimv2_vision_model", "Aimv2VisionConfig"),
+ ("albert", "AlbertConfig"),
+ ("align", "AlignConfig"),
+ ("align_text_model", "AlignTextConfig"),
+ ("align_vision_model", "AlignVisionConfig"),
+ ("altclip", "AltCLIPConfig"),
+ ("altclip_text_model", "AltCLIPTextConfig"),
+ ("altclip_vision_model", "AltCLIPVisionConfig"),
+ ("apertus", "ApertusConfig"),
+ ("arcee", "ArceeConfig"),
+ ("aria", "AriaConfig"),
+ ("aria_text", "AriaTextConfig"),
+ ("audio-spectrogram-transformer", "ASTConfig"),
+ ("audioflamingo3", "AudioFlamingo3Config"),
+ ("audioflamingo3_encoder", "AudioFlamingo3EncoderConfig"),
+ ("autoformer", "AutoformerConfig"),
+ ("aya_vision", "AyaVisionConfig"),
+ ("bamba", "BambaConfig"),
+ ("bark", "BarkConfig"),
+ ("bart", "BartConfig"),
+ ("beit", "BeitConfig"),
+ ("bert", "BertConfig"),
+ ("bert-generation", "BertGenerationConfig"),
+ ("big_bird", "BigBirdConfig"),
+ ("bigbird_pegasus", "BigBirdPegasusConfig"),
+ ("biogpt", "BioGptConfig"),
+ ("bit", "BitConfig"),
+ ("bitnet", "BitNetConfig"),
+ ("blenderbot", "BlenderbotConfig"),
+ ("blenderbot-small", "BlenderbotSmallConfig"),
+ ("blip", "BlipConfig"),
+ ("blip-2", "Blip2Config"),
+ ("blip_2_qformer", "Blip2QFormerConfig"),
+ ("blip_2_vision_model", "Blip2VisionConfig"),
+ ("blip_text_model", "BlipTextConfig"),
+ ("blip_vision_model", "BlipVisionConfig"),
+ ("bloom", "BloomConfig"),
+ ("blt", "BltConfig"),
+ ("blt_global_transformer", "BltGlobalTransformerConfig"),
+ ("blt_local_decoder", "BltLocalDecoderConfig"),
+ ("blt_local_encoder", "BltLocalEncoderConfig"),
+ ("blt_patcher", "BltPatcherConfig"),
+ ("bridgetower", "BridgeTowerConfig"),
+ ("bridgetower_text_model", "BridgeTowerTextConfig"),
+ ("bridgetower_vision_model", "BridgeTowerVisionConfig"),
+ ("bros", "BrosConfig"),
+ ("camembert", "CamembertConfig"),
+ ("canine", "CanineConfig"),
+ ("chameleon", "ChameleonConfig"),
+ ("chameleon_vqgan", "ChameleonVQVAEConfig"),
+ ("chinese_clip", "ChineseCLIPConfig"),
+ ("chinese_clip_text_model", "ChineseCLIPTextConfig"),
+ ("chinese_clip_vision_model", "ChineseCLIPVisionConfig"),
+ ("chmv2", "CHMv2Config"),
+ ("clap", "ClapConfig"),
+ ("clap_audio_model", "ClapAudioConfig"),
+ ("clap_text_model", "ClapTextConfig"),
+ ("clip", "CLIPConfig"),
+ ("clip_text_model", "CLIPTextConfig"),
+ ("clip_vision_model", "CLIPVisionConfig"),
+ ("clipseg", "CLIPSegConfig"),
+ ("clipseg_text_model", "CLIPSegTextConfig"),
+ ("clipseg_vision_model", "CLIPSegVisionConfig"),
+ ("clvp", "ClvpConfig"),
+ ("clvp_decoder", "ClvpDecoderConfig"),
+ ("clvp_encoder", "ClvpEncoderConfig"),
+ ("codegen", "CodeGenConfig"),
+ ("cohere", "CohereConfig"),
+ ("cohere2", "Cohere2Config"),
+ ("cohere2_moe", "Cohere2MoeConfig"),
+ ("cohere2_vision", "Cohere2VisionConfig"),
+ ("cohere_asr", "CohereAsrConfig"),
+ ("colmodernvbert", "ColModernVBertConfig"),
+ ("colpali", "ColPaliConfig"),
+ ("colqwen2", "ColQwen2Config"),
+ ("conditional_detr", "ConditionalDetrConfig"),
+ ("convbert", "ConvBertConfig"),
+ ("convnext", "ConvNextConfig"),
+ ("convnextv2", "ConvNextV2Config"),
+ ("cpmant", "CpmAntConfig"),
+ ("csm", "CsmConfig"),
+ ("csm_depth_decoder_model", "CsmDepthDecoderConfig"),
+ ("ctrl", "CTRLConfig"),
+ ("cvt", "CvtConfig"),
+ ("cwm", "CwmConfig"),
+ ("d_fine", "DFineConfig"),
+ ("dab-detr", "DabDetrConfig"),
+ ("dac", "DacConfig"),
+ ("data2vec-audio", "Data2VecAudioConfig"),
+ ("data2vec-text", "Data2VecTextConfig"),
+ ("data2vec-vision", "Data2VecVisionConfig"),
+ ("dbrx", "DbrxConfig"),
+ ("deberta", "DebertaConfig"),
+ ("deberta-v2", "DebertaV2Config"),
+ ("decision_transformer", "DecisionTransformerConfig"),
+ ("deepseek_v2", "DeepseekV2Config"),
+ ("deepseek_v3", "DeepseekV3Config"),
+ ("deepseek_v4", "DeepseekV4Config"),
+ ("deepseek_vl", "DeepseekVLConfig"),
+ ("deepseek_vl_hybrid", "DeepseekVLHybridConfig"),
+ ("deformable_detr", "DeformableDetrConfig"),
+ ("deimv2", "Deimv2Config"),
+ ("deit", "DeiTConfig"),
+ ("depth_anything", "DepthAnythingConfig"),
+ ("depth_pro", "DepthProConfig"),
+ ("detr", "DetrConfig"),
+ ("dia", "DiaConfig"),
+ ("dia_decoder", "DiaDecoderConfig"),
+ ("dia_encoder", "DiaEncoderConfig"),
+ ("diffllama", "DiffLlamaConfig"),
+ ("dinat", "DinatConfig"),
+ ("dinov2", "Dinov2Config"),
+ ("dinov2_with_registers", "Dinov2WithRegistersConfig"),
+ ("dinov3_convnext", "DINOv3ConvNextConfig"),
+ ("dinov3_vit", "DINOv3ViTConfig"),
+ ("distilbert", "DistilBertConfig"),
+ ("doge", "DogeConfig"),
+ ("donut-swin", "DonutSwinConfig"),
+ ("dots1", "Dots1Config"),
+ ("dpr", "DPRConfig"),
+ ("dpt", "DPTConfig"),
+ ("edgetam", "EdgeTamConfig"),
+ ("edgetam_video", "EdgeTamVideoConfig"),
+ ("edgetam_vision_model", "EdgeTamVisionConfig"),
+ ("efficientloftr", "EfficientLoFTRConfig"),
+ ("efficientnet", "EfficientNetConfig"),
+ ("electra", "ElectraConfig"),
+ ("emu3", "Emu3Config"),
+ ("emu3_text_model", "Emu3TextConfig"),
+ ("emu3_vqgan", "Emu3VQVAEConfig"),
+ ("encodec", "EncodecConfig"),
+ ("encoder-decoder", "EncoderDecoderConfig"),
+ ("eomt", "EomtConfig"),
+ ("eomt_dinov3", "EomtDinov3Config"),
+ ("ernie", "ErnieConfig"),
+ ("ernie4_5", "Ernie4_5Config"),
+ ("ernie4_5_moe", "Ernie4_5_MoeConfig"),
+ ("ernie4_5_vl_moe", "Ernie4_5_VLMoeConfig"),
+ ("ernie4_5_vl_moe_text", "Ernie4_5_VLMoeTextConfig"),
+ ("ernie4_5_vl_moe_vision", "Ernie4_5_VLMoeVisionConfig"),
+ ("esm", "EsmConfig"),
+ ("eurobert", "EuroBertConfig"),
+ ("evolla", "EvollaConfig"),
+ ("exaone4", "Exaone4Config"),
+ ("exaone4_5", "Exaone4_5_Config"),
+ ("exaone4_5_vision", "Exaone4_5_VisionConfig"),
+ ("exaone_moe", "ExaoneMoeConfig"),
+ ("falcon", "FalconConfig"),
+ ("falcon_h1", "FalconH1Config"),
+ ("falcon_mamba", "FalconMambaConfig"),
+ ("fast_vlm", "FastVlmConfig"),
+ ("fastspeech2_conformer", "FastSpeech2ConformerConfig"),
+ ("fastspeech2_conformer_hifigan", "FastSpeech2ConformerHifiGanConfig"),
+ ("fastspeech2_conformer_with_hifigan", "FastSpeech2ConformerWithHifiGanConfig"),
+ ("flaubert", "FlaubertConfig"),
+ ("flava", "FlavaConfig"),
+ ("flava_image_model", "FlavaImageConfig"),
+ ("flava_multimodal_model", "FlavaMultimodalConfig"),
+ ("flava_text_model", "FlavaTextConfig"),
+ ("flex_olmo", "FlexOlmoConfig"),
+ ("florence2", "Florence2Config"),
+ ("florence_vision", "Florence2VisionConfig"),
+ ("fnet", "FNetConfig"),
+ ("focalnet", "FocalNetConfig"),
+ ("fsmt", "FSMTConfig"),
+ ("funnel", "FunnelConfig"),
+ ("fuyu", "FuyuConfig"),
+ ("gemma", "GemmaConfig"),
+ ("gemma2", "Gemma2Config"),
+ ("gemma3", "Gemma3Config"),
+ ("gemma3_text", "Gemma3TextConfig"),
+ ("gemma3n", "Gemma3nConfig"),
+ ("gemma3n_audio", "Gemma3nAudioConfig"),
+ ("gemma3n_text", "Gemma3nTextConfig"),
+ ("gemma3n_vision", "Gemma3nVisionConfig"),
+ ("gemma4", "Gemma4Config"),
+ ("gemma4_assistant", "Gemma4AssistantConfig"),
+ ("gemma4_audio", "Gemma4AudioConfig"),
+ ("gemma4_text", "Gemma4TextConfig"),
+ ("gemma4_vision", "Gemma4VisionConfig"),
+ ("git", "GitConfig"),
+ ("git_vision_model", "GitVisionConfig"),
+ ("glm", "GlmConfig"),
+ ("glm4", "Glm4Config"),
+ ("glm46v", "Glm46VConfig"),
+ ("glm4_moe", "Glm4MoeConfig"),
+ ("glm4_moe_lite", "Glm4MoeLiteConfig"),
+ ("glm4v", "Glm4vConfig"),
+ ("glm4v_moe", "Glm4vMoeConfig"),
+ ("glm4v_moe_text", "Glm4vMoeTextConfig"),
+ ("glm4v_moe_vision", "Glm4vMoeVisionConfig"),
+ ("glm4v_text", "Glm4vTextConfig"),
+ ("glm4v_vision", "Glm4vVisionConfig"),
+ ("glm_image", "GlmImageConfig"),
+ ("glm_image_text", "GlmImageTextConfig"),
+ ("glm_image_vision", "GlmImageVisionConfig"),
+ ("glm_image_vqmodel", "GlmImageVQVAEConfig"),
+ ("glm_moe_dsa", "GlmMoeDsaConfig"),
+ ("glm_ocr", "GlmOcrConfig"),
+ ("glm_ocr_text", "GlmOcrTextConfig"),
+ ("glm_ocr_vision", "GlmOcrVisionConfig"),
+ ("glmasr", "GlmAsrConfig"),
+ ("glmasr_encoder", "GlmAsrEncoderConfig"),
+ ("glpn", "GLPNConfig"),
+ ("got_ocr2", "GotOcr2Config"),
+ ("gpt2", "GPT2Config"),
+ ("gpt_bigcode", "GPTBigCodeConfig"),
+ ("gpt_neo", "GPTNeoConfig"),
+ ("gpt_neox", "GPTNeoXConfig"),
+ ("gpt_neox_japanese", "GPTNeoXJapaneseConfig"),
+ ("gpt_oss", "GptOssConfig"),
+ ("gptj", "GPTJConfig"),
+ ("granite", "GraniteConfig"),
+ ("granite4_vision", "Granite4VisionConfig"),
+ ("granite4_vision_text", "Granite4VisionTextConfig"),
+ ("granite_speech", "GraniteSpeechConfig"),
+ ("granite_speech_encoder", "GraniteSpeechEncoderConfig"),
+ ("granite_speech_plus", "GraniteSpeechPlusConfig"),
+ ("granite_speech_plus_encoder", "GraniteSpeechPlusEncoderConfig"),
+ ("granitemoe", "GraniteMoeConfig"),
+ ("granitemoehybrid", "GraniteMoeHybridConfig"),
+ ("granitemoeshared", "GraniteMoeSharedConfig"),
+ ("grounding-dino", "GroundingDinoConfig"),
+ ("groupvit", "GroupViTConfig"),
+ ("groupvit_text_model", "GroupViTTextConfig"),
+ ("groupvit_vision_model", "GroupViTVisionConfig"),
+ ("helium", "HeliumConfig"),
+ ("hgnet_v2", "HGNetV2Config"),
+ ("hiera", "HieraConfig"),
+ ("higgs_audio_v2", "HiggsAudioV2Config"),
+ ("higgs_audio_v2_tokenizer", "HiggsAudioV2TokenizerConfig"),
+ ("hrm_text", "HrmTextConfig"),
+ ("hubert", "HubertConfig"),
+ ("hunyuan_v1_dense", "HunYuanDenseV1Config"),
+ ("hunyuan_v1_moe", "HunYuanMoEV1Config"),
+ ("hy_v3", "HYV3Config"),
+ ("hyperclovax", "HyperCLOVAXConfig"),
+ ("ibert", "IBertConfig"),
+ ("idefics", "IdeficsConfig"),
+ ("idefics2", "Idefics2Config"),
+ ("idefics2_perceiver", "Idefics2PerceiverConfig"),
+ ("idefics2_vision", "Idefics2VisionConfig"),
+ ("idefics3", "Idefics3Config"),
+ ("idefics3_vision", "Idefics3VisionConfig"),
+ ("idefics_perciever", "IdeficsPerceiverConfig"),
+ ("idefics_vision", "IdeficsVisionConfig"),
+ ("ijepa", "IJepaConfig"),
+ ("imagegpt", "ImageGPTConfig"),
+ ("informer", "InformerConfig"),
+ ("instructblip", "InstructBlipConfig"),
+ ("instructblip_qformer", "InstructBlipQFormerConfig"),
+ ("instructblip_vision_model", "InstructBlipVisionConfig"),
+ ("instructblipvideo", "InstructBlipVideoConfig"),
+ ("instructblipvideo_qformer", "InstructBlipVideoQFormerConfig"),
+ ("instructblipvideo_vision_model", "InstructBlipVideoVisionConfig"),
+ ("internvl", "InternVLConfig"),
+ ("internvl_vision", "InternVLVisionConfig"),
+ ("jais2", "Jais2Config"),
+ ("jamba", "JambaConfig"),
+ ("janus", "JanusConfig"),
+ ("janus_vision_model", "JanusVisionConfig"),
+ ("janus_vqgan", "JanusVQVAEConfig"),
+ ("jetmoe", "JetMoeConfig"),
+ ("jina_embeddings_v3", "JinaEmbeddingsV3Config"),
+ ("kosmos-2", "Kosmos2Config"),
+ ("kosmos-2.5", "Kosmos2_5Config"),
+ ("kosmos_2_5_text_model", "Kosmos2_5TextConfig"),
+ ("kosmos_2_5_vision_model", "Kosmos2_5VisionConfig"),
+ ("kosmos_2_text_model", "Kosmos2TextConfig"),
+ ("kosmos_2_vision_model", "Kosmos2VisionConfig"),
+ ("kyutai_speech_to_text", "KyutaiSpeechToTextConfig"),
+ ("laguna", "LagunaConfig"),
+ ("lasr_ctc", "LasrCTCConfig"),
+ ("lasr_encoder", "LasrEncoderConfig"),
+ ("layoutlm", "LayoutLMConfig"),
+ ("layoutlmv2", "LayoutLMv2Config"),
+ ("layoutlmv3", "LayoutLMv3Config"),
+ ("layoutxlm", "LayoutXLMConfig"),
+ ("led", "LEDConfig"),
+ ("levit", "LevitConfig"),
+ ("lfm2", "Lfm2Config"),
+ ("lfm2_moe", "Lfm2MoeConfig"),
+ ("lfm2_vl", "Lfm2VlConfig"),
+ ("lightglue", "LightGlueConfig"),
+ ("lighton_ocr", "LightOnOcrConfig"),
+ ("lilt", "LiltConfig"),
+ ("llama", "LlamaConfig"),
+ ("llama4", "Llama4Config"),
+ ("llama4_text", "Llama4TextConfig"),
+ ("llama4_vision_model", "Llama4VisionConfig"),
+ ("llava", "LlavaConfig"),
+ ("llava_next", "LlavaNextConfig"),
+ ("llava_next_video", "LlavaNextVideoConfig"),
+ ("llava_onevision", "LlavaOnevisionConfig"),
+ ("longcat_flash", "LongcatFlashConfig"),
+ ("longformer", "LongformerConfig"),
+ ("longt5", "LongT5Config"),
+ ("luke", "LukeConfig"),
+ ("lw_detr", "LwDetrConfig"),
+ ("lw_detr_vit", "LwDetrViTConfig"),
+ ("lxmert", "LxmertConfig"),
+ ("m2m_100", "M2M100Config"),
+ ("mamba", "MambaConfig"),
+ ("mamba2", "Mamba2Config"),
+ ("marian", "MarianConfig"),
+ ("markuplm", "MarkupLMConfig"),
+ ("mask2former", "Mask2FormerConfig"),
+ ("maskformer", "MaskFormerConfig"),
+ ("maskformer-swin", "MaskFormerSwinConfig"),
+ ("mbart", "MBartConfig"),
+ ("megatron-bert", "MegatronBertConfig"),
+ ("metaclip_2", "MetaClip2Config"),
+ ("metaclip_2_text_model", "MetaClip2TextConfig"),
+ ("metaclip_2_vision_model", "MetaClip2VisionConfig"),
+ ("mgp-str", "MgpstrConfig"),
+ ("mimi", "MimiConfig"),
+ ("minicpmv4_6", "MiniCPMV4_6Config"),
+ ("minicpmv4_6_vision", "MiniCPMV4_6VisionConfig"),
+ ("minimax", "MiniMaxConfig"),
+ ("minimax_m2", "MiniMaxM2Config"),
+ ("ministral", "MinistralConfig"),
+ ("ministral3", "Ministral3Config"),
+ ("mistral", "MistralConfig"),
+ ("mistral3", "Mistral3Config"),
+ ("mistral4", "Mistral4Config"),
+ ("mixtral", "MixtralConfig"),
+ ("mlcd_vision_model", "MLCDVisionConfig"),
+ ("mllama", "MllamaConfig"),
+ ("mllama_text_model", "MllamaTextConfig"),
+ ("mllama_vision_model", "MllamaVisionConfig"),
+ ("mm-grounding-dino", "MMGroundingDinoConfig"),
+ ("mobilebert", "MobileBertConfig"),
+ ("mobilenet_v1", "MobileNetV1Config"),
+ ("mobilenet_v2", "MobileNetV2Config"),
+ ("mobilevit", "MobileViTConfig"),
+ ("mobilevitv2", "MobileViTV2Config"),
+ ("modernbert", "ModernBertConfig"),
+ ("modernbert-decoder", "ModernBertDecoderConfig"),
+ ("modernvbert", "ModernVBertConfig"),
+ ("moonshine", "MoonshineConfig"),
+ ("moonshine_streaming", "MoonshineStreamingConfig"),
+ ("moonshine_streaming_encoder", "MoonshineStreamingEncoderConfig"),
+ ("moshi", "MoshiConfig"),
+ ("moshi_depth", "MoshiDepthConfig"),
+ ("mpnet", "MPNetConfig"),
+ ("mpt", "MptConfig"),
+ ("mra", "MraConfig"),
+ ("mt5", "MT5Config"),
+ ("musicflamingo", "MusicFlamingoConfig"),
+ ("musicgen", "MusicgenConfig"),
+ ("musicgen_decoder", "MusicgenDecoderConfig"),
+ ("musicgen_melody", "MusicgenMelodyConfig"),
+ ("musicgen_melody_decoder", "MusicgenMelodyDecoderConfig"),
+ ("mvp", "MvpConfig"),
+ ("nanochat", "NanoChatConfig"),
+ ("nemotron", "NemotronConfig"),
+ ("nemotron_h", "NemotronHConfig"),
+ ("nllb-moe", "NllbMoeConfig"),
+ ("nomic_bert", "NomicBertConfig"),
+ ("nougat", "NougatConfig"),
+ ("nystromformer", "NystromformerConfig"),
+ ("olmo", "OlmoConfig"),
+ ("olmo2", "Olmo2Config"),
+ ("olmo3", "Olmo3Config"),
+ ("olmo_hybrid", "OlmoHybridConfig"),
+ ("olmoe", "OlmoeConfig"),
+ ("omdet-turbo", "OmDetTurboConfig"),
+ ("oneformer", "OneFormerConfig"),
+ ("openai-gpt", "OpenAIGPTConfig"),
+ ("openai_privacy_filter", "OpenAIPrivacyFilterConfig"),
+ ("opt", "OPTConfig"),
+ ("ovis2", "Ovis2Config"),
+ ("owlv2", "Owlv2Config"),
+ ("owlv2_text_model", "Owlv2TextConfig"),
+ ("owlv2_vision_model", "Owlv2VisionConfig"),
+ ("owlvit", "OwlViTConfig"),
+ ("owlvit_text_model", "OwlViTTextConfig"),
+ ("owlvit_vision_model", "OwlViTVisionConfig"),
+ ("paddleocr_vl", "PaddleOCRVLConfig"),
+ ("paddleocr_vl_text", "PaddleOCRTextConfig"),
+ ("paddleocr_vl_vision", "PaddleOCRVisionConfig"),
+ ("paligemma", "PaliGemmaConfig"),
+ ("parakeet_ctc", "ParakeetCTCConfig"),
+ ("parakeet_encoder", "ParakeetEncoderConfig"),
+ ("parakeet_tdt", "ParakeetTDTConfig"),
+ ("patchtsmixer", "PatchTSMixerConfig"),
+ ("patchtst", "PatchTSTConfig"),
+ ("pe_audio", "PeAudioConfig"),
+ ("pe_audio_encoder", "PeAudioEncoderConfig"),
+ ("pe_audio_video", "PeAudioVideoConfig"),
+ ("pe_audio_video_encoder", "PeAudioVideoEncoderConfig"),
+ ("pe_video", "PeVideoConfig"),
+ ("pe_video_encoder", "PeVideoEncoderConfig"),
+ ("pegasus", "PegasusConfig"),
+ ("pegasus_x", "PegasusXConfig"),
+ ("perceiver", "PerceiverConfig"),
+ ("perception_lm", "PerceptionLMConfig"),
+ ("persimmon", "PersimmonConfig"),
+ ("phi", "PhiConfig"),
+ ("phi3", "Phi3Config"),
+ ("phi4_multimodal", "Phi4MultimodalConfig"),
+ ("phi4_multimodal_audio", "Phi4MultimodalAudioConfig"),
+ ("phi4_multimodal_vision", "Phi4MultimodalVisionConfig"),
+ ("phimoe", "PhimoeConfig"),
+ ("pi0", "PI0Config"),
+ ("pix2struct", "Pix2StructConfig"),
+ ("pix2struct_text_model", "Pix2StructTextConfig"),
+ ("pix2struct_vision_model", "Pix2StructVisionConfig"),
+ ("pixio", "PixioConfig"),
+ ("pixtral", "PixtralVisionConfig"),
+ ("plbart", "PLBartConfig"),
+ ("poolformer", "PoolFormerConfig"),
+ ("pop2piano", "Pop2PianoConfig"),
+ ("pp_chart2table", "PPChart2TableConfig"),
+ ("pp_doclayout_v2", "PPDocLayoutV2Config"),
+ ("pp_doclayout_v3", "PPDocLayoutV3Config"),
+ ("pp_formulanet", "PPFormulaNetConfig"),
+ ("pp_lcnet", "PPLCNetConfig"),
+ ("pp_lcnet_v3", "PPLCNetV3Config"),
+ ("pp_ocrv5_mobile_det", "PPOCRV5MobileDetConfig"),
+ ("pp_ocrv5_mobile_rec", "PPOCRV5MobileRecConfig"),
+ ("pp_ocrv5_server_det", "PPOCRV5ServerDetConfig"),
+ ("pp_ocrv5_server_rec", "PPOCRV5ServerRecConfig"),
+ ("prompt_depth_anything", "PromptDepthAnythingConfig"),
+ ("prophetnet", "ProphetNetConfig"),
+ ("pvt", "PvtConfig"),
+ ("pvt_v2", "PvtV2Config"),
+ ("qianfan_ocr", "QianfanOCRConfig"),
+ ("qianfan_ocr_vision", "QianfanOCRVisionConfig"),
+ ("qwen2", "Qwen2Config"),
+ ("qwen2_5_omni", "Qwen2_5OmniConfig"),
+ ("qwen2_5_omni_audio_encoder", "Qwen2_5OmniAudioEncoderConfig"),
+ ("qwen2_5_omni_bigvgan", "Qwen2_5OmniBigVGANConfig"),
+ ("qwen2_5_omni_dit", "Qwen2_5OmniDiTConfig"),
+ ("qwen2_5_omni_talker", "Qwen2_5OmniTalkerConfig"),
+ ("qwen2_5_omni_text", "Qwen2_5OmniTextConfig"),
+ ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerConfig"),
+ ("qwen2_5_omni_token2wav", "Qwen2_5OmniToken2WavConfig"),
+ ("qwen2_5_omni_vision_encoder", "Qwen2_5OmniVisionEncoderConfig"),
+ ("qwen2_5_vl", "Qwen2_5_VLConfig"),
+ ("qwen2_5_vl_text", "Qwen2_5_VLTextConfig"),
+ ("qwen2_5_vl_vision", "Qwen2_5_VLVisionConfig"),
+ ("qwen2_audio", "Qwen2AudioConfig"),
+ ("qwen2_audio_encoder", "Qwen2AudioEncoderConfig"),
+ ("qwen2_moe", "Qwen2MoeConfig"),
+ ("qwen2_vl", "Qwen2VLConfig"),
+ ("qwen2_vl_text", "Qwen2VLTextConfig"),
+ ("qwen2_vl_vision", "Qwen2VLVisionConfig"),
+ ("qwen3", "Qwen3Config"),
+ ("qwen3_5", "Qwen3_5Config"),
+ ("qwen3_5_moe", "Qwen3_5MoeConfig"),
+ ("qwen3_5_moe_text", "Qwen3_5MoeTextConfig"),
+ ("qwen3_5_moe_vision", "Qwen3_5MoeVisionConfig"),
+ ("qwen3_5_text", "Qwen3_5TextConfig"),
+ ("qwen3_5_vision", "Qwen3_5VisionConfig"),
+ ("qwen3_moe", "Qwen3MoeConfig"),
+ ("qwen3_next", "Qwen3NextConfig"),
+ ("qwen3_omni_moe", "Qwen3OmniMoeConfig"),
+ ("qwen3_omni_moe_audio_encoder", "Qwen3OmniMoeAudioEncoderConfig"),
+ ("qwen3_omni_moe_talker_code_predictor", "Qwen3OmniMoeTalkerCodePredictorConfig"),
+ ("qwen3_omni_moe_talker_text", "Qwen3OmniMoeTalkerTextConfig"),
+ ("qwen3_omni_moe_text", "Qwen3OmniMoeTextConfig"),
+ ("qwen3_omni_moe_thinker", "Qwen3OmniMoeThinkerConfig"),
+ ("qwen3_omni_moe_vision_encoder", "Qwen3OmniMoeVisionEncoderConfig"),
+ ("qwen3_vl", "Qwen3VLConfig"),
+ ("qwen3_vl_moe", "Qwen3VLMoeConfig"),
+ ("qwen3_vl_moe_text", "Qwen3VLMoeTextConfig"),
+ ("qwen3_vl_moe_vision", "Qwen3VLMoeVisionConfig"),
+ ("qwen3_vl_text", "Qwen3VLTextConfig"),
+ ("qwen3_vl_vision", "Qwen3VLVisionConfig"),
+ ("rag", "RagConfig"),
+ ("recurrent_gemma", "RecurrentGemmaConfig"),
+ ("reformer", "ReformerConfig"),
+ ("regnet", "RegNetConfig"),
+ ("rembert", "RemBertConfig"),
+ ("resnet", "ResNetConfig"),
+ ("rf_detr", "RfDetrConfig"),
+ ("rf_detr_dinov2", "RfDetrDinov2Config"),
+ ("roberta", "RobertaConfig"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormConfig"),
+ ("roc_bert", "RoCBertConfig"),
+ ("roformer", "RoFormerConfig"),
+ ("rt_detr", "RTDetrConfig"),
+ ("rt_detr_resnet", "RTDetrResNetConfig"),
+ ("rt_detr_v2", "RTDetrV2Config"),
+ ("rwkv", "RwkvConfig"),
+ ("sam", "SamConfig"),
+ ("sam2", "Sam2Config"),
+ ("sam2_hiera_det_model", "Sam2HieraDetConfig"),
+ ("sam2_video", "Sam2VideoConfig"),
+ ("sam2_vision_model", "Sam2VisionConfig"),
+ ("sam3", "Sam3Config"),
+ ("sam3_detr_decoder", "Sam3DETRDecoderConfig"),
+ ("sam3_detr_encoder", "Sam3DETREncoderConfig"),
+ ("sam3_geometry_encoder", "Sam3GeometryEncoderConfig"),
+ ("sam3_lite_text", "Sam3LiteTextConfig"),
+ ("sam3_lite_text_detr_decoder", "Sam3LiteTextDETRDecoderConfig"),
+ ("sam3_lite_text_detr_encoder", "Sam3LiteTextDETREncoderConfig"),
+ ("sam3_lite_text_geometry_encoder", "Sam3LiteTextGeometryEncoderConfig"),
+ ("sam3_lite_text_mask_decoder", "Sam3LiteTextMaskDecoderConfig"),
+ ("sam3_lite_text_text_model", "Sam3LiteTextTextConfig"),
+ ("sam3_mask_decoder", "Sam3MaskDecoderConfig"),
+ ("sam3_tracker", "Sam3TrackerConfig"),
+ ("sam3_tracker_video", "Sam3TrackerVideoConfig"),
+ ("sam3_video", "Sam3VideoConfig"),
+ ("sam3_vision_model", "Sam3VisionConfig"),
+ ("sam3_vit_model", "Sam3ViTConfig"),
+ ("sam_hq", "SamHQConfig"),
+ ("sam_hq_vision_model", "SamHQVisionConfig"),
+ ("sam_vision_model", "SamVisionConfig"),
+ ("seamless_m4t", "SeamlessM4TConfig"),
+ ("seamless_m4t_v2", "SeamlessM4Tv2Config"),
+ ("seed_oss", "SeedOssConfig"),
+ ("segformer", "SegformerConfig"),
+ ("seggpt", "SegGptConfig"),
+ ("sew", "SEWConfig"),
+ ("sew-d", "SEWDConfig"),
+ ("shieldgemma2", "ShieldGemma2Config"),
+ ("siglip", "SiglipConfig"),
+ ("siglip2", "Siglip2Config"),
+ ("siglip2_text_model", "Siglip2TextConfig"),
+ ("siglip2_vision_model", "Siglip2VisionConfig"),
+ ("siglip_text_model", "SiglipTextConfig"),
+ ("siglip_vision_model", "SiglipVisionConfig"),
+ ("slanet", "SLANetConfig"),
+ ("slanext", "SLANeXtConfig"),
+ ("smollm3", "SmolLM3Config"),
+ ("smolvlm", "SmolVLMConfig"),
+ ("smolvlm_vision", "SmolVLMVisionConfig"),
+ ("solar_open", "SolarOpenConfig"),
+ ("speech-encoder-decoder", "SpeechEncoderDecoderConfig"),
+ ("speech_to_text", "Speech2TextConfig"),
+ ("speecht5", "SpeechT5Config"),
+ ("speecht5_hifigan", "SpeechT5HifiGanConfig"),
+ ("splinter", "SplinterConfig"),
+ ("squeezebert", "SqueezeBertConfig"),
+ ("stablelm", "StableLmConfig"),
+ ("starcoder2", "Starcoder2Config"),
+ ("superglue", "SuperGlueConfig"),
+ ("superpoint", "SuperPointConfig"),
+ ("swiftformer", "SwiftFormerConfig"),
+ ("swin", "SwinConfig"),
+ ("swin2sr", "Swin2SRConfig"),
+ ("swinv2", "Swinv2Config"),
+ ("switch_transformers", "SwitchTransformersConfig"),
+ ("t5", "T5Config"),
+ ("t5_gemma_module", "T5GemmaModuleConfig"),
+ ("t5gemma", "T5GemmaConfig"),
+ ("t5gemma2", "T5Gemma2Config"),
+ ("t5gemma2_decoder", "T5Gemma2DecoderConfig"),
+ ("t5gemma2_encoder", "T5Gemma2EncoderConfig"),
+ ("t5gemma2_text", "T5Gemma2TextConfig"),
+ ("table-transformer", "TableTransformerConfig"),
+ ("tapas", "TapasConfig"),
+ ("textnet", "TextNetConfig"),
+ ("time_series_transformer", "TimeSeriesTransformerConfig"),
+ ("timesfm", "TimesFmConfig"),
+ ("timesfm2_5", "TimesFm2_5Config"),
+ ("timesformer", "TimesformerConfig"),
+ ("timm_backbone", "TimmBackboneConfig"),
+ ("timm_wrapper", "TimmWrapperConfig"),
+ ("trocr", "TrOCRConfig"),
+ ("tvp", "TvpConfig"),
+ ("udop", "UdopConfig"),
+ ("umt5", "UMT5Config"),
+ ("unispeech", "UniSpeechConfig"),
+ ("unispeech-sat", "UniSpeechSatConfig"),
+ ("univnet", "UnivNetConfig"),
+ ("upernet", "UperNetConfig"),
+ ("uvdoc", "UVDocConfig"),
+ ("uvdoc_backbone", "UVDocBackboneConfig"),
+ ("vaultgemma", "VaultGemmaConfig"),
+ ("vibevoice_acoustic_tokenizer", "VibeVoiceAcousticTokenizerConfig"),
+ ("vibevoice_asr", "VibeVoiceAsrConfig"),
+ ("video_llama_3", "VideoLlama3Config"),
+ ("video_llama_3_vision", "VideoLlama3VisionConfig"),
+ ("video_llava", "VideoLlavaConfig"),
+ ("videomae", "VideoMAEConfig"),
+ ("videomt", "VideomtConfig"),
+ ("vilt", "ViltConfig"),
+ ("vipllava", "VipLlavaConfig"),
+ ("vision-encoder-decoder", "VisionEncoderDecoderConfig"),
+ ("vision-text-dual-encoder", "VisionTextDualEncoderConfig"),
+ ("visual_bert", "VisualBertConfig"),
+ ("vit", "ViTConfig"),
+ ("vit_mae", "ViTMAEConfig"),
+ ("vit_msn", "ViTMSNConfig"),
+ ("vitdet", "VitDetConfig"),
+ ("vitmatte", "VitMatteConfig"),
+ ("vitpose", "VitPoseConfig"),
+ ("vitpose_backbone", "VitPoseBackboneConfig"),
+ ("vits", "VitsConfig"),
+ ("vivit", "VivitConfig"),
+ ("vjepa2", "VJEPA2Config"),
+ ("voxtral", "VoxtralConfig"),
+ ("voxtral_encoder", "VoxtralEncoderConfig"),
+ ("voxtral_realtime", "VoxtralRealtimeConfig"),
+ ("voxtral_realtime_encoder", "VoxtralRealtimeEncoderConfig"),
+ ("voxtral_realtime_text", "VoxtralRealtimeTextConfig"),
+ ("wav2vec2", "Wav2Vec2Config"),
+ ("wav2vec2-bert", "Wav2Vec2BertConfig"),
+ ("wav2vec2-conformer", "Wav2Vec2ConformerConfig"),
+ ("wavlm", "WavLMConfig"),
+ ("whisper", "WhisperConfig"),
+ ("xclip", "XCLIPConfig"),
+ ("xclip_text_model", "XCLIPTextConfig"),
+ ("xclip_vision_model", "XCLIPVisionConfig"),
+ ("xcodec", "XcodecConfig"),
+ ("xglm", "XGLMConfig"),
+ ("xlm", "XLMConfig"),
+ ("xlm-roberta", "XLMRobertaConfig"),
+ ("xlm-roberta-xl", "XLMRobertaXLConfig"),
+ ("xlnet", "XLNetConfig"),
+ ("xlstm", "xLSTMConfig"),
+ ("xmod", "XmodConfig"),
+ ("yolos", "YolosConfig"),
+ ("yoso", "YosoConfig"),
+ ("youtu", "YoutuConfig"),
+ ("zamba", "ZambaConfig"),
+ ("zamba2", "Zamba2Config"),
+ ("zoedepth", "ZoeDepthConfig"),
+ ]
+)
+
+SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict(
+ [
+ ("aimv2_text_model", "aimv2"),
+ ("aimv2_vision_model", "aimv2"),
+ ("align_text_model", "align"),
+ ("align_vision_model", "align"),
+ ("altclip_text_model", "altclip"),
+ ("altclip_vision_model", "altclip"),
+ ("aria_text", "aria"),
+ ("audio-spectrogram-transformer", "audio_spectrogram_transformer"),
+ ("audioflamingo3_encoder", "audioflamingo3"),
+ ("bert-generation", "bert_generation"),
+ ("blenderbot-small", "blenderbot_small"),
+ ("blip-2", "blip_2"),
+ ("blip_2_qformer", "blip_2"),
+ ("blip_2_vision_model", "blip_2"),
+ ("blip_text_model", "blip"),
+ ("blip_vision_model", "blip"),
+ ("blt_global_transformer", "blt"),
+ ("blt_local_decoder", "blt"),
+ ("blt_local_encoder", "blt"),
+ ("blt_patcher", "blt"),
+ ("bridgetower_text_model", "bridgetower"),
+ ("bridgetower_vision_model", "bridgetower"),
+ ("chameleon_vqgan", "chameleon"),
+ ("chinese_clip_text_model", "chinese_clip"),
+ ("chinese_clip_vision_model", "chinese_clip"),
+ ("clap_audio_model", "clap"),
+ ("clap_text_model", "clap"),
+ ("clip_text_model", "clip"),
+ ("clip_vision_model", "clip"),
+ ("clipseg_text_model", "clipseg"),
+ ("clipseg_vision_model", "clipseg"),
+ ("clvp_decoder", "clvp"),
+ ("clvp_encoder", "clvp"),
+ ("csm_depth_decoder_model", "csm"),
+ ("dab-detr", "dab_detr"),
+ ("data2vec-audio", "data2vec"),
+ ("data2vec-text", "data2vec"),
+ ("data2vec-vision", "data2vec"),
+ ("deberta-v2", "deberta_v2"),
+ ("dia_decoder", "dia"),
+ ("dia_encoder", "dia"),
+ ("donut-swin", "donut"),
+ ("edgetam_vision_model", "edgetam"),
+ ("emu3_text_model", "emu3"),
+ ("emu3_vqgan", "emu3"),
+ ("encoder-decoder", "encoder_decoder"),
+ ("ernie4_5_vl_moe_text", "ernie4_5_vl_moe"),
+ ("ernie4_5_vl_moe_vision", "ernie4_5_vl_moe"),
+ ("exaone4_5_vision", "exaone4_5"),
+ ("fastspeech2_conformer_hifigan", "fastspeech2_conformer"),
+ ("fastspeech2_conformer_with_hifigan", "fastspeech2_conformer"),
+ ("flava_image_model", "flava"),
+ ("flava_multimodal_model", "flava"),
+ ("flava_text_model", "flava"),
+ ("florence_vision", "florence2"),
+ ("gemma3_text", "gemma3"),
+ ("gemma3n_audio", "gemma3n"),
+ ("gemma3n_text", "gemma3n"),
+ ("gemma3n_vision", "gemma3n"),
+ ("gemma4_audio", "gemma4"),
+ ("gemma4_text", "gemma4"),
+ ("gemma4_vision", "gemma4"),
+ ("git_vision_model", "git"),
+ ("glm4v_moe_text", "glm4v_moe"),
+ ("glm4v_moe_vision", "glm4v_moe"),
+ ("glm4v_text", "glm4v"),
+ ("glm4v_vision", "glm4v"),
+ ("glm_image_text", "glm_image"),
+ ("glm_image_vision", "glm_image"),
+ ("glm_image_vqmodel", "glm_image"),
+ ("glm_ocr_text", "glm_ocr"),
+ ("glm_ocr_vision", "glm_ocr"),
+ ("glmasr_encoder", "glmasr"),
+ ("granite4_vision_text", "granite4_vision"),
+ ("granite_speech_encoder", "granite_speech"),
+ ("granite_speech_plus_encoder", "granite_speech_plus"),
+ ("grounding-dino", "grounding_dino"),
+ ("groupvit_text_model", "groupvit"),
+ ("groupvit_vision_model", "groupvit"),
+ ("idefics2_perceiver", "idefics2"),
+ ("idefics2_vision", "idefics2"),
+ ("idefics3_vision", "idefics3"),
+ ("idefics_perciever", "idefics"),
+ ("idefics_vision", "idefics"),
+ ("instructblip_qformer", "instructblip"),
+ ("instructblip_vision_model", "instructblip"),
+ ("instructblipvideo_qformer", "instructblipvideo"),
+ ("instructblipvideo_vision_model", "instructblipvideo"),
+ ("internvl_vision", "internvl"),
+ ("janus_vision_model", "janus"),
+ ("janus_vqgan", "janus"),
+ ("kosmos-2", "kosmos2"),
+ ("kosmos-2.5", "kosmos2_5"),
+ ("kosmos_2_5_text_model", "kosmos2_5"),
+ ("kosmos_2_5_vision_model", "kosmos2_5"),
+ ("kosmos_2_text_model", "kosmos2"),
+ ("kosmos_2_vision_model", "kosmos2"),
+ ("lasr_ctc", "lasr"),
+ ("lasr_encoder", "lasr"),
+ ("llama4_text", "llama4"),
+ ("llama4_vision_model", "llama4"),
+ ("lw_detr_vit", "lw_detr"),
+ ("maskformer-swin", "maskformer"),
+ ("megatron-bert", "megatron_bert"),
+ ("metaclip_2_text_model", "metaclip_2"),
+ ("metaclip_2_vision_model", "metaclip_2"),
+ ("mgp-str", "mgp_str"),
+ ("minicpmv4_6_vision", "minicpmv4_6"),
+ ("mlcd_vision_model", "mlcd"),
+ ("mllama_text_model", "mllama"),
+ ("mllama_vision_model", "mllama"),
+ ("mm-grounding-dino", "mm_grounding_dino"),
+ ("modernbert-decoder", "modernbert_decoder"),
+ ("moonshine_streaming_encoder", "moonshine_streaming"),
+ ("moshi_depth", "moshi"),
+ ("musicgen_decoder", "musicgen"),
+ ("musicgen_melody_decoder", "musicgen_melody"),
+ ("nllb-moe", "nllb_moe"),
+ ("omdet-turbo", "omdet_turbo"),
+ ("openai-gpt", "openai"),
+ ("owlv2_text_model", "owlv2"),
+ ("owlv2_vision_model", "owlv2"),
+ ("owlvit_text_model", "owlvit"),
+ ("owlvit_vision_model", "owlvit"),
+ ("paddleocr_vl_text", "paddleocr_vl"),
+ ("paddleocr_vl_vision", "paddleocr_vl"),
+ ("parakeet_ctc", "parakeet"),
+ ("parakeet_encoder", "parakeet"),
+ ("parakeet_tdt", "parakeet"),
+ ("pe_audio_encoder", "pe_audio"),
+ ("pe_audio_video_encoder", "pe_audio_video"),
+ ("pe_video_encoder", "pe_video"),
+ ("phi4_multimodal_audio", "phi4_multimodal"),
+ ("phi4_multimodal_vision", "phi4_multimodal"),
+ ("pix2struct_text_model", "pix2struct"),
+ ("pix2struct_vision_model", "pix2struct"),
+ ("qianfan_ocr_vision", "qianfan_ocr"),
+ ("qwen2_5_omni_audio_encoder", "qwen2_5_omni"),
+ ("qwen2_5_omni_bigvgan", "qwen2_5_omni"),
+ ("qwen2_5_omni_dit", "qwen2_5_omni"),
+ ("qwen2_5_omni_talker", "qwen2_5_omni"),
+ ("qwen2_5_omni_text", "qwen2_5_omni"),
+ ("qwen2_5_omni_thinker", "qwen2_5_omni"),
+ ("qwen2_5_omni_token2wav", "qwen2_5_omni"),
+ ("qwen2_5_omni_vision_encoder", "qwen2_5_omni"),
+ ("qwen2_5_vl_text", "qwen2_5_vl"),
+ ("qwen2_5_vl_vision", "qwen2_5_vl"),
+ ("qwen2_audio_encoder", "qwen2_audio"),
+ ("qwen2_vl_text", "qwen2_vl"),
+ ("qwen2_vl_vision", "qwen2_vl"),
+ ("qwen3_5_moe_text", "qwen3_5_moe"),
+ ("qwen3_5_moe_vision", "qwen3_5_moe"),
+ ("qwen3_5_text", "qwen3_5"),
+ ("qwen3_5_vision", "qwen3_5"),
+ ("qwen3_omni_moe_audio_encoder", "qwen3_omni_moe"),
+ ("qwen3_omni_moe_talker_code_predictor", "qwen3_omni_moe"),
+ ("qwen3_omni_moe_talker_text", "qwen3_omni_moe"),
+ ("qwen3_omni_moe_text", "qwen3_omni_moe"),
+ ("qwen3_omni_moe_thinker", "qwen3_omni_moe"),
+ ("qwen3_omni_moe_vision_encoder", "qwen3_omni_moe"),
+ ("qwen3_vl_moe_text", "qwen3_vl_moe"),
+ ("qwen3_vl_moe_vision", "qwen3_vl_moe"),
+ ("qwen3_vl_text", "qwen3_vl"),
+ ("qwen3_vl_vision", "qwen3_vl"),
+ ("rf_detr_dinov2", "rf_detr"),
+ ("roberta-prelayernorm", "roberta_prelayernorm"),
+ ("rt_detr_resnet", "rt_detr"),
+ ("sam2_hiera_det_model", "sam2"),
+ ("sam2_vision_model", "sam2"),
+ ("sam3_detr_decoder", "sam3"),
+ ("sam3_detr_encoder", "sam3"),
+ ("sam3_geometry_encoder", "sam3"),
+ ("sam3_lite_text_detr_decoder", "sam3_lite_text"),
+ ("sam3_lite_text_detr_encoder", "sam3_lite_text"),
+ ("sam3_lite_text_geometry_encoder", "sam3_lite_text"),
+ ("sam3_lite_text_mask_decoder", "sam3_lite_text"),
+ ("sam3_lite_text_text_model", "sam3_lite_text"),
+ ("sam3_mask_decoder", "sam3"),
+ ("sam3_vision_model", "sam3"),
+ ("sam3_vit_model", "sam3"),
+ ("sam_hq_vision_model", "sam_hq"),
+ ("sam_vision_model", "sam"),
+ ("sew-d", "sew_d"),
+ ("siglip2_text_model", "siglip2"),
+ ("siglip2_vision_model", "siglip2"),
+ ("siglip_text_model", "siglip"),
+ ("siglip_vision_model", "siglip"),
+ ("smolvlm_vision", "smolvlm"),
+ ("speech-encoder-decoder", "speech_encoder_decoder"),
+ ("speecht5_hifigan", "speecht5"),
+ ("t5_gemma_module", "t5gemma"),
+ ("t5gemma2_decoder", "t5gemma2"),
+ ("t5gemma2_encoder", "t5gemma2"),
+ ("t5gemma2_text", "t5gemma2"),
+ ("table-transformer", "table_transformer"),
+ ("unispeech-sat", "unispeech_sat"),
+ ("uvdoc_backbone", "uvdoc"),
+ ("video_llama_3_vision", "video_llama_3"),
+ ("vision-encoder-decoder", "vision_encoder_decoder"),
+ ("vision-text-dual-encoder", "vision_text_dual_encoder"),
+ ("voxtral_encoder", "voxtral"),
+ ("voxtral_realtime_encoder", "voxtral_realtime"),
+ ("voxtral_realtime_text", "voxtral_realtime"),
+ ("wav2vec2-bert", "wav2vec2_bert"),
+ ("wav2vec2-conformer", "wav2vec2_conformer"),
+ ("xclip", "x_clip"),
+ ("xclip_text_model", "x_clip"),
+ ("xclip_vision_model", "x_clip"),
+ ("xlm-roberta", "xlm_roberta"),
+ ("xlm-roberta-xl", "xlm_roberta_xl"),
+ ]
+)
+
+IMAGE_PROCESSOR_MAPPING_NAMES = OrderedDict(
+ [
+ ("aria", {"pil": "AriaImageProcessorPil", "torchvision": "AriaImageProcessor"}),
+ ("beit", {"pil": "BeitImageProcessorPil", "torchvision": "BeitImageProcessor"}),
+ ("bit", {"pil": "BitImageProcessorPil", "torchvision": "BitImageProcessor"}),
+ ("blip", {"pil": "BlipImageProcessorPil", "torchvision": "BlipImageProcessor"}),
+ ("bridgetower", {"pil": "BridgeTowerImageProcessorPil", "torchvision": "BridgeTowerImageProcessor"}),
+ ("chameleon", {"pil": "ChameleonImageProcessorPil", "torchvision": "ChameleonImageProcessor"}),
+ ("chinese_clip", {"pil": "ChineseCLIPImageProcessorPil", "torchvision": "ChineseCLIPImageProcessor"}),
+ ("chmv2", {"torchvision": "CHMv2ImageProcessor"}),
+ ("clip", {"pil": "CLIPImageProcessorPil", "torchvision": "CLIPImageProcessor"}),
+ ("cohere2_vision", {"torchvision": "Cohere2VisionImageProcessor"}),
+ (
+ "conditional_detr",
+ {"pil": "ConditionalDetrImageProcessorPil", "torchvision": "ConditionalDetrImageProcessor"},
+ ),
+ ("convnext", {"pil": "ConvNextImageProcessorPil", "torchvision": "ConvNextImageProcessor"}),
+ ("deepseek_vl", {"pil": "DeepseekVLImageProcessorPil", "torchvision": "DeepseekVLImageProcessor"}),
+ (
+ "deepseek_vl_hybrid",
+ {"pil": "DeepseekVLHybridImageProcessorPil", "torchvision": "DeepseekVLHybridImageProcessor"},
+ ),
+ ("deformable_detr", {"pil": "DeformableDetrImageProcessorPil", "torchvision": "DeformableDetrImageProcessor"}),
+ ("deit", {"pil": "DeiTImageProcessorPil", "torchvision": "DeiTImageProcessor"}),
+ ("depth_pro", {"torchvision": "DepthProImageProcessor"}),
+ ("detr", {"pil": "DetrImageProcessorPil", "torchvision": "DetrImageProcessor"}),
+ ("dinov3_vit", {"torchvision": "DINOv3ViTImageProcessor"}),
+ ("dpt", {"pil": "DPTImageProcessorPil", "torchvision": "DPTImageProcessor"}),
+ ("efficientloftr", {"pil": "EfficientLoFTRImageProcessorPil", "torchvision": "EfficientLoFTRImageProcessor"}),
+ ("efficientnet", {"pil": "EfficientNetImageProcessorPil", "torchvision": "EfficientNetImageProcessor"}),
+ ("eomt", {"pil": "EomtImageProcessorPil", "torchvision": "EomtImageProcessor"}),
+ ("ernie4_5_vl_moe", {"pil": "Ernie4_5_VLMoeImageProcessorPil", "torchvision": "Ernie4_5_VLMoeImageProcessor"}),
+ ("flava", {"pil": "FlavaImageProcessorPil", "torchvision": "FlavaImageProcessor"}),
+ ("fuyu", {"pil": "FuyuImageProcessorPil", "torchvision": "FuyuImageProcessor"}),
+ ("gemma3", {"pil": "Gemma3ImageProcessorPil", "torchvision": "Gemma3ImageProcessor"}),
+ ("gemma4", {"pil": "Gemma4ImageProcessorPil", "torchvision": "Gemma4ImageProcessor"}),
+ ("glm46v", {"pil": "Glm46VImageProcessorPil", "torchvision": "Glm46VImageProcessor"}),
+ ("glm4v", {"pil": "Glm4vImageProcessorPil", "torchvision": "Glm4vImageProcessor"}),
+ ("glm_image", {"pil": "GlmImageImageProcessorPil", "torchvision": "GlmImageImageProcessor"}),
+ ("glpn", {"pil": "GLPNImageProcessorPil", "torchvision": "GLPNImageProcessor"}),
+ ("got_ocr2", {"pil": "GotOcr2ImageProcessorPil", "torchvision": "GotOcr2ImageProcessor"}),
+ ("grounding-dino", {"pil": "GroundingDinoImageProcessorPil", "torchvision": "GroundingDinoImageProcessor"}),
+ ("idefics", {"pil": "IdeficsImageProcessorPil", "torchvision": "IdeficsImageProcessor"}),
+ ("idefics2", {"pil": "Idefics2ImageProcessorPil", "torchvision": "Idefics2ImageProcessor"}),
+ ("idefics3", {"pil": "Idefics3ImageProcessorPil", "torchvision": "Idefics3ImageProcessor"}),
+ ("imagegpt", {"pil": "ImageGPTImageProcessorPil", "torchvision": "ImageGPTImageProcessor"}),
+ ("janus", {"pil": "JanusImageProcessorPil", "torchvision": "JanusImageProcessor"}),
+ ("layoutlmv2", {"pil": "LayoutLMv2ImageProcessorPil", "torchvision": "LayoutLMv2ImageProcessor"}),
+ ("layoutlmv3", {"pil": "LayoutLMv3ImageProcessorPil", "torchvision": "LayoutLMv3ImageProcessor"}),
+ ("levit", {"pil": "LevitImageProcessorPil", "torchvision": "LevitImageProcessor"}),
+ ("lfm2_vl", {"torchvision": "Lfm2VlImageProcessor"}),
+ ("lightglue", {"pil": "LightGlueImageProcessorPil", "torchvision": "LightGlueImageProcessor"}),
+ ("llama4", {"torchvision": "Llama4ImageProcessor"}),
+ ("llava", {"pil": "LlavaImageProcessorPil", "torchvision": "LlavaImageProcessor"}),
+ ("llava_next", {"pil": "LlavaNextImageProcessorPil", "torchvision": "LlavaNextImageProcessor"}),
+ ("llava_onevision", {"pil": "LlavaOnevisionImageProcessorPil", "torchvision": "LlavaOnevisionImageProcessor"}),
+ ("mask2former", {"pil": "Mask2FormerImageProcessorPil", "torchvision": "Mask2FormerImageProcessor"}),
+ ("maskformer", {"pil": "MaskFormerImageProcessorPil", "torchvision": "MaskFormerImageProcessor"}),
+ ("minicpmv4_6", {"pil": "MiniCPMV4_6ImageProcessorPil", "torchvision": "MiniCPMV4_6ImageProcessor"}),
+ ("mllama", {"pil": "MllamaImageProcessorPil", "torchvision": "MllamaImageProcessor"}),
+ ("mobilenet_v1", {"pil": "MobileNetV1ImageProcessorPil", "torchvision": "MobileNetV1ImageProcessor"}),
+ ("mobilenet_v2", {"pil": "MobileNetV2ImageProcessorPil", "torchvision": "MobileNetV2ImageProcessor"}),
+ ("mobilevit", {"pil": "MobileViTImageProcessorPil", "torchvision": "MobileViTImageProcessor"}),
+ ("nougat", {"pil": "NougatImageProcessorPil", "torchvision": "NougatImageProcessor"}),
+ ("oneformer", {"pil": "OneFormerImageProcessorPil", "torchvision": "OneFormerImageProcessor"}),
+ ("ovis2", {"pil": "Ovis2ImageProcessorPil", "torchvision": "Ovis2ImageProcessor"}),
+ ("owlv2", {"pil": "Owlv2ImageProcessorPil", "torchvision": "Owlv2ImageProcessor"}),
+ ("owlvit", {"pil": "OwlViTImageProcessorPil", "torchvision": "OwlViTImageProcessor"}),
+ ("paddleocr_vl", {"pil": "PaddleOCRVLImageProcessorPil", "torchvision": "PaddleOCRVLImageProcessor"}),
+ ("perceiver", {"pil": "PerceiverImageProcessorPil", "torchvision": "PerceiverImageProcessor"}),
+ ("perception_lm", {"torchvision": "PerceptionLMImageProcessor"}),
+ ("phi4_multimodal", {"torchvision": "Phi4MultimodalImageProcessor"}),
+ ("pi0", {"torchvision": "PI0ImageProcessor"}),
+ ("pix2struct", {"pil": "Pix2StructImageProcessorPil", "torchvision": "Pix2StructImageProcessor"}),
+ ("pixtral", {"pil": "PixtralImageProcessorPil", "torchvision": "PixtralImageProcessor"}),
+ ("poolformer", {"pil": "PoolFormerImageProcessorPil", "torchvision": "PoolFormerImageProcessor"}),
+ ("pp_chart2table", {"pil": "PPChart2TableImageProcessorPil", "torchvision": "PPChart2TableImageProcessor"}),
+ ("pp_doclayout_v2", {"torchvision": "PPDocLayoutV2ImageProcessor"}),
+ ("pp_doclayout_v3", {"torchvision": "PPDocLayoutV3ImageProcessor"}),
+ ("pp_formulanet", {"torchvision": "PPFormulaNetImageProcessor"}),
+ ("pp_lcnet", {"torchvision": "PPLCNetImageProcessor"}),
+ ("pp_ocrv5_server_det", {"torchvision": "PPOCRV5ServerDetImageProcessor"}),
+ ("pp_ocrv5_server_rec", {"torchvision": "PPOCRV5ServerRecImageProcessor"}),
+ (
+ "prompt_depth_anything",
+ {"pil": "PromptDepthAnythingImageProcessorPil", "torchvision": "PromptDepthAnythingImageProcessor"},
+ ),
+ ("pvt", {"pil": "PvtImageProcessorPil", "torchvision": "PvtImageProcessor"}),
+ ("qwen2_vl", {"pil": "Qwen2VLImageProcessorPil", "torchvision": "Qwen2VLImageProcessor"}),
+ ("rf_detr", {"torchvision": "RfDetrImageProcessor"}),
+ ("rt_detr", {"pil": "RTDetrImageProcessorPil", "torchvision": "RTDetrImageProcessor"}),
+ ("sam", {"pil": "SamImageProcessorPil", "torchvision": "SamImageProcessor"}),
+ ("sam2", {"torchvision": "Sam2ImageProcessor"}),
+ ("sam3", {"torchvision": "Sam3ImageProcessor"}),
+ ("segformer", {"pil": "SegformerImageProcessorPil", "torchvision": "SegformerImageProcessor"}),
+ ("seggpt", {"pil": "SegGptImageProcessorPil", "torchvision": "SegGptImageProcessor"}),
+ ("siglip", {"pil": "SiglipImageProcessorPil", "torchvision": "SiglipImageProcessor"}),
+ ("siglip2", {"pil": "Siglip2ImageProcessorPil", "torchvision": "Siglip2ImageProcessor"}),
+ ("slanext", {"torchvision": "SLANeXtImageProcessor"}),
+ ("smolvlm", {"pil": "SmolVLMImageProcessorPil", "torchvision": "SmolVLMImageProcessor"}),
+ ("superglue", {"pil": "SuperGlueImageProcessorPil", "torchvision": "SuperGlueImageProcessor"}),
+ ("superpoint", {"pil": "SuperPointImageProcessorPil", "torchvision": "SuperPointImageProcessor"}),
+ ("swin2sr", {"pil": "Swin2SRImageProcessorPil", "torchvision": "Swin2SRImageProcessor"}),
+ ("textnet", {"pil": "TextNetImageProcessorPil", "torchvision": "TextNetImageProcessor"}),
+ ("tvp", {"pil": "TvpImageProcessorPil", "torchvision": "TvpImageProcessor"}),
+ ("uvdoc", {"torchvision": "UVDocImageProcessor"}),
+ ("video_llama_3", {"pil": "VideoLlama3ImageProcessorPil", "torchvision": "VideoLlama3ImageProcessor"}),
+ ("videomae", {"pil": "VideoMAEImageProcessorPil", "torchvision": "VideoMAEImageProcessor"}),
+ ("vilt", {"pil": "ViltImageProcessorPil", "torchvision": "ViltImageProcessor"}),
+ ("vit", {"pil": "ViTImageProcessorPil", "torchvision": "ViTImageProcessor"}),
+ ("vitmatte", {"pil": "VitMatteImageProcessorPil", "torchvision": "VitMatteImageProcessor"}),
+ ("vitpose", {"pil": "VitPoseImageProcessorPil", "torchvision": "VitPoseImageProcessor"}),
+ ("yolos", {"pil": "YolosImageProcessorPil", "torchvision": "YolosImageProcessor"}),
+ ("zoedepth", {"pil": "ZoeDepthImageProcessorPil", "torchvision": "ZoeDepthImageProcessor"}),
+ ]
+)
+
+VIDEO_PROCESSOR_MAPPING_NAMES = OrderedDict(
+ [
+ ("ernie4_5_vl_moe", "Ernie4_5_VLMoeVideoProcessor"),
+ ("gemma4", "Gemma4VideoProcessor"),
+ ("glm46v", "Glm46VVideoProcessor"),
+ ("glm4v", "Glm4vVideoProcessor"),
+ ("instructblipvideo", "InstructBlipVideoVideoProcessor"),
+ ("internvl", "InternVLVideoProcessor"),
+ ("llava_next_video", "LlavaNextVideoVideoProcessor"),
+ ("llava_onevision", "LlavaOnevisionVideoProcessor"),
+ ("minicpmv4_6", "MiniCPMV4_6VideoProcessor"),
+ ("pe_video", "PeVideoVideoProcessor"),
+ ("perception_lm", "PerceptionLMVideoProcessor"),
+ ("qwen2_vl", "Qwen2VLVideoProcessor"),
+ ("qwen3_vl", "Qwen3VLVideoProcessor"),
+ ("sam2_video", "Sam2VideoVideoProcessor"),
+ ("smolvlm", "SmolVLMVideoProcessor"),
+ ("video_llama_3", "VideoLlama3VideoProcessor"),
+ ("video_llava", "VideoLlavaVideoProcessor"),
+ ("videomae", "VideoMAEVideoProcessor"),
+ ("videomt", "VideomtVideoProcessor"),
+ ("vjepa2", "VJEPA2VideoProcessor"),
+ ]
+)
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/feature_extraction_auto.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/feature_extraction_auto.py
new file mode 100644
index 0000000000000000000000000000000000000000..953a8e7cf74261bfc4ce93d48b99be498afead77
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/feature_extraction_auto.py
@@ -0,0 +1,388 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""AutoFeatureExtractor class."""
+
+import importlib
+import os
+from collections import OrderedDict
+
+# Build the list of all feature extractors
+from ...configuration_utils import PreTrainedConfig
+from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
+from ...feature_extraction_utils import FeatureExtractionMixin
+from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, cached_file, logging, safe_load_json_file
+from .auto_factory import _LazyAutoMapping
+from .configuration_auto import (
+ CONFIG_MAPPING_NAMES,
+ AutoConfig,
+ model_type_to_module_name,
+ replace_list_option_in_docstrings,
+)
+
+
+logger = logging.get_logger(__name__)
+
+FEATURE_EXTRACTOR_MAPPING_NAMES = OrderedDict(
+ [
+ ("audio-spectrogram-transformer", "ASTFeatureExtractor"),
+ ("audioflamingo3", "WhisperFeatureExtractor"),
+ ("clap", "ClapFeatureExtractor"),
+ ("clvp", "ClvpFeatureExtractor"),
+ ("cohere_asr", "CohereAsrFeatureExtractor"),
+ ("csm", "EncodecFeatureExtractor"),
+ ("dac", "DacFeatureExtractor"),
+ ("data2vec-audio", "Wav2Vec2FeatureExtractor"),
+ ("dia", "DiaFeatureExtractor"),
+ ("encodec", "EncodecFeatureExtractor"),
+ ("gemma3n", "Gemma3nAudioFeatureExtractor"),
+ ("gemma4", "Gemma4AudioFeatureExtractor"),
+ ("glmasr", "WhisperFeatureExtractor"),
+ ("granite_speech", "GraniteSpeechFeatureExtractor"),
+ ("granite_speech_plus", "GraniteSpeechFeatureExtractor"),
+ ("higgs_audio_v2_tokenizer", "DacFeatureExtractor"),
+ ("hubert", "Wav2Vec2FeatureExtractor"),
+ ("kyutai_speech_to_text", "KyutaiSpeechToTextFeatureExtractor"),
+ ("lasr_ctc", "LasrFeatureExtractor"),
+ ("lasr_encoder", "LasrFeatureExtractor"),
+ ("markuplm", "MarkupLMFeatureExtractor"),
+ ("mimi", "EncodecFeatureExtractor"),
+ ("moonshine", "Wav2Vec2FeatureExtractor"),
+ ("moshi", "EncodecFeatureExtractor"),
+ ("musicgen", "EncodecFeatureExtractor"),
+ ("musicgen_melody", "MusicgenMelodyFeatureExtractor"),
+ ("parakeet_ctc", "ParakeetFeatureExtractor"),
+ ("parakeet_encoder", "ParakeetFeatureExtractor"),
+ ("parakeet_tdt", "ParakeetFeatureExtractor"),
+ ("pe_audio", "PeAudioFeatureExtractor"),
+ ("pe_audio_video", "PeAudioFeatureExtractor"),
+ ("phi4_multimodal", "Phi4MultimodalFeatureExtractor"),
+ ("pop2piano", "Pop2PianoFeatureExtractor"),
+ ("qwen2_5_omni", "WhisperFeatureExtractor"),
+ ("qwen2_audio", "WhisperFeatureExtractor"),
+ ("qwen3_omni_moe", "WhisperFeatureExtractor"),
+ ("seamless_m4t", "SeamlessM4TFeatureExtractor"),
+ ("seamless_m4t_v2", "SeamlessM4TFeatureExtractor"),
+ ("sew", "Wav2Vec2FeatureExtractor"),
+ ("sew-d", "Wav2Vec2FeatureExtractor"),
+ ("speech_to_text", "Speech2TextFeatureExtractor"),
+ ("speecht5", "SpeechT5FeatureExtractor"),
+ ("unispeech", "Wav2Vec2FeatureExtractor"),
+ ("unispeech-sat", "Wav2Vec2FeatureExtractor"),
+ ("univnet", "UnivNetFeatureExtractor"),
+ ("vibevoice_acoustic_tokenizer", "VibeVoiceAcousticTokenizerFeatureExtractor"),
+ ("vibevoice_asr", "VibeVoiceAcousticTokenizerFeatureExtractor"),
+ ("voxtral", "WhisperFeatureExtractor"),
+ ("voxtral_realtime", "VoxtralRealtimeFeatureExtractor"),
+ ("wav2vec2", "Wav2Vec2FeatureExtractor"),
+ ("wav2vec2-bert", "Wav2Vec2FeatureExtractor"),
+ ("wav2vec2-conformer", "Wav2Vec2FeatureExtractor"),
+ ("wavlm", "Wav2Vec2FeatureExtractor"),
+ ("whisper", "WhisperFeatureExtractor"),
+ ("xcodec", "DacFeatureExtractor"),
+ ]
+)
+
+FEATURE_EXTRACTOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)
+
+
+def feature_extractor_class_from_name(class_name: str):
+ for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
+ if class_name in extractors:
+ module_name = model_type_to_module_name(module_name)
+
+ module = importlib.import_module(f".{module_name}", "transformers.models")
+ try:
+ return getattr(module, class_name)
+ except AttributeError:
+ continue
+
+ for extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.values():
+ if getattr(extractor, "__name__", None) == class_name:
+ return extractor
+
+ # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
+ # init and we return the proper dummy to get an appropriate error message.
+ main_module = importlib.import_module("transformers")
+ if hasattr(main_module, class_name):
+ return getattr(main_module, class_name)
+
+ return None
+
+
+def get_feature_extractor_config(
+ pretrained_model_name_or_path: str | os.PathLike,
+ cache_dir: str | os.PathLike | None = None,
+ force_download: bool = False,
+ proxies: dict[str, str] | None = None,
+ token: bool | str | None = None,
+ revision: str | None = None,
+ local_files_only: bool = False,
+ **kwargs,
+):
+ """
+ Loads the feature extractor configuration from a pretrained model feature extractor configuration.
+
+ Args:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a configuration file saved using the
+ [`~FeatureExtractionMixin.save_pretrained`] method, e.g., `./my_model_directory/`.
+
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
+ cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the configuration files and override the cached versions if they
+ exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ local_files_only (`bool`, *optional*, defaults to `False`):
+ If `True`, will only try to load the feature extractor configuration from local files.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Returns:
+ `Dict`: The configuration of the feature extractor.
+
+ Examples:
+
+ ```python
+ # Download configuration from huggingface.co and cache.
+ feature_extractor_config = get_feature_extractor_config("facebook/wav2vec2-base-960h")
+ # This model does not have a feature extractor config so the result will be an empty dict.
+ feature_extractor_config = get_feature_extractor_config("FacebookAI/xlm-roberta-base")
+
+ # Save a pretrained feature extractor locally and you can reload its config
+ from transformers import AutoFeatureExtractor
+
+ feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h")
+ feature_extractor.save_pretrained("feature-extractor-test")
+ feature_extractor_config = get_feature_extractor_config("feature-extractor-test")
+ ```"""
+ # Load with a priority given to the nested processor config, if available in repo
+ resolved_processor_file = cached_file(
+ pretrained_model_name_or_path,
+ filename=PROCESSOR_NAME,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ )
+ resolved_feature_extractor_file = cached_file(
+ pretrained_model_name_or_path,
+ filename=FEATURE_EXTRACTOR_NAME,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ )
+
+ # An empty list if none of the possible files is found in the repo
+ if not resolved_feature_extractor_file and not resolved_processor_file:
+ logger.info("Could not locate the feature extractor configuration file.")
+ return {}
+
+ # Load feature_extractor dict. Priority goes as (nested config if found -> feature extractor config)
+ # We are downloading both configs because almost all models have a `processor_config.json` but
+ # not all of these are nested. We need to check if it was saved recently as nested or if it is legacy style
+ feature_extractor_dict = {}
+ if resolved_processor_file is not None:
+ processor_dict = safe_load_json_file(resolved_processor_file)
+ if "feature_extractor" in processor_dict:
+ feature_extractor_dict = processor_dict["feature_extractor"]
+
+ if resolved_feature_extractor_file is not None and feature_extractor_dict is None:
+ feature_extractor_dict = safe_load_json_file(resolved_feature_extractor_file)
+ return feature_extractor_dict
+
+
+class AutoFeatureExtractor:
+ r"""
+ This is a generic feature extractor class that will be instantiated as one of the feature extractor classes of the
+ library when created with the [`AutoFeatureExtractor.from_pretrained`] class method.
+
+ This class cannot be instantiated directly using `__init__()` (throws an error).
+ """
+
+ def __init__(self):
+ raise OSError(
+ "AutoFeatureExtractor is designed to be instantiated "
+ "using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method."
+ )
+
+ @classmethod
+ @replace_list_option_in_docstrings(FEATURE_EXTRACTOR_MAPPING_NAMES)
+ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+ r"""
+ Instantiate one of the feature extractor classes of the library from a pretrained model vocabulary.
+
+ The feature extractor class to instantiate is selected based on the `model_type` property of the config object
+ (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's
+ missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
+
+ List options
+
+ Params:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a feature extractor file saved using the
+ [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] method, e.g.,
+ `./my_model_directory/`.
+ - a path to a saved feature extractor JSON *file*, e.g.,
+ `./my_model_directory/preprocessor_config.json`.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model feature extractor should be cached if the
+ standard cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the feature extractor files and override the cached versions
+ if they exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ return_unused_kwargs (`bool`, *optional*, defaults to `False`):
+ If `False`, then this function returns just the final feature extractor object. If `True`, then this
+ functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of
+ `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
+ trust_remote_code (`bool`, *optional*, defaults to `False`):
+ Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
+ kwargs (`dict[str, Any]`, *optional*):
+ The values in kwargs of any keys which are feature extractor attributes will be used to override the
+ loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is
+ controlled by the `return_unused_kwargs` keyword parameter.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoFeatureExtractor
+
+ >>> # Download feature extractor from huggingface.co and cache.
+ >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h")
+
+ >>> # If feature extractor files are in a directory (e.g. feature extractor was saved using *save_pretrained('./test/saved_model/')*)
+ >>> # feature_extractor = AutoFeatureExtractor.from_pretrained("./test/saved_model/")
+ ```"""
+ config = kwargs.pop("config", None)
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
+ kwargs["_from_auto"] = True
+
+ config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs)
+ feature_extractor_class = config_dict.get("feature_extractor_type", None)
+ feature_extractor_auto_map = None
+ if "AutoFeatureExtractor" in config_dict.get("auto_map", {}):
+ feature_extractor_auto_map = config_dict["auto_map"]["AutoFeatureExtractor"]
+
+ # If we don't find the feature extractor class in the feature extractor config, let's try the model config.
+ if feature_extractor_class is None and feature_extractor_auto_map is None:
+ if not isinstance(config, PreTrainedConfig):
+ config = AutoConfig.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+ # It could be in `config.feature_extractor_type``
+ feature_extractor_class = getattr(config, "feature_extractor_type", None)
+ if hasattr(config, "auto_map") and "AutoFeatureExtractor" in config.auto_map:
+ feature_extractor_auto_map = config.auto_map["AutoFeatureExtractor"]
+
+ if feature_extractor_class is not None:
+ feature_extractor_class = feature_extractor_class_from_name(feature_extractor_class)
+
+ has_remote_code = feature_extractor_auto_map is not None
+ has_local_code = feature_extractor_class is not None or type(config) in FEATURE_EXTRACTOR_MAPPING
+ explicit_local_code = has_local_code and not (
+ feature_extractor_class or FEATURE_EXTRACTOR_MAPPING[type(config)]
+ ).__module__.startswith("transformers.")
+ if has_remote_code:
+ if "--" in feature_extractor_auto_map:
+ upstream_repo = feature_extractor_auto_map.split("--")[0]
+ else:
+ upstream_repo = None
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
+ )
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ feature_extractor_class = get_class_from_dynamic_module(
+ feature_extractor_auto_map, pretrained_model_name_or_path, **kwargs
+ )
+ _ = kwargs.pop("code_revision", None)
+ feature_extractor_class.register_for_auto_class()
+ return feature_extractor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
+ elif feature_extractor_class is not None:
+ return feature_extractor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
+ # Last try: we use the FEATURE_EXTRACTOR_MAPPING.
+ elif type(config) in FEATURE_EXTRACTOR_MAPPING:
+ feature_extractor_class = FEATURE_EXTRACTOR_MAPPING[type(config)]
+ return feature_extractor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+ raise ValueError(
+ f"Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a "
+ f"`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following "
+ f"`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES)}"
+ )
+
+ @staticmethod
+ def register(config_class, feature_extractor_class, exist_ok=False):
+ """
+ Register a new feature extractor for this class.
+
+ Args:
+ config_class ([`PreTrainedConfig`]):
+ The configuration corresponding to the model to register.
+ feature_extractor_class ([`FeatureExtractorMixin`]): The feature extractor to register.
+ """
+ FEATURE_EXTRACTOR_MAPPING.register(config_class, feature_extractor_class, exist_ok=exist_ok)
+
+
+__all__ = ["FEATURE_EXTRACTOR_MAPPING", "AutoFeatureExtractor"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/image_processing_auto.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/image_processing_auto.py
new file mode 100644
index 0000000000000000000000000000000000000000..56e8acf9db71f32724a9455f67e70878f04c81fa
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/image_processing_auto.py
@@ -0,0 +1,706 @@
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""AutoImageProcessor class."""
+
+import importlib
+import os
+from collections import OrderedDict
+from typing import TYPE_CHECKING
+
+# Build the list of all image processors
+from ...configuration_utils import PreTrainedConfig
+from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
+from ...image_processing_utils import ImageProcessingMixin
+from ...utils import (
+ CONFIG_NAME,
+ IMAGE_PROCESSOR_NAME,
+ PROCESSOR_NAME,
+ cached_file,
+ is_timm_config_dict,
+ is_timm_local_checkpoint,
+ is_torchvision_available,
+ logging,
+ safe_load_json_file,
+)
+from ...utils.import_utils import requires
+from .auto_factory import _LazyAutoMapping
+from .auto_mappings import IMAGE_PROCESSOR_MAPPING_NAMES
+from .configuration_auto import (
+ CONFIG_MAPPING_NAMES,
+ AutoConfig,
+ model_type_to_module_name,
+ replace_list_option_in_docstrings,
+)
+
+
+logger = logging.get_logger(__name__)
+
+# These image processors use Lanczos interpolation, which is not supported by fast image processors.
+# To avoid important differences in outputs, we default to using the PIL backend for these processors.
+DEFAULT_TO_PIL_BACKEND_IMAGE_PROCESSORS = [
+ "ChameleonImageProcessor",
+ "FlavaImageProcessor",
+ "Idefics3ImageProcessor",
+ "SmolVLMImageProcessor",
+]
+
+
+if TYPE_CHECKING:
+ # This significantly improves completion suggestion performance when
+ # the transformers package is used with Microsoft's Pylance language server.
+ IMAGE_PROCESSOR_MAPPING_NAMES: OrderedDict[str, dict[str, str | None]] = OrderedDict()
+else:
+ # Merge non-standard mapping names with auto-inferred `IMAGE_PROCESSOR_MAPPING_NAMES`
+ MISSING_IMAGE_PROCESSOR_MAPPING_NAMES = OrderedDict(
+ [
+ ("aimv2", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("aimv2_vision_model", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("align", {"torchvision": "EfficientNetImageProcessor", "pil": "EfficientNetImageProcessorPil"}),
+ ("altclip", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("aya_vision", {"torchvision": "GotOcr2ImageProcessor", "pil": "GotOcr2ImageProcessorPil"}),
+ ("blip-2", {"torchvision": "BlipImageProcessor", "pil": "BlipImageProcessorPil"}),
+ ("clipseg", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("colpali", {"torchvision": "SiglipImageProcessor", "pil": "SiglipImageProcessorPil"}),
+ ("colqwen2", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("convnextv2", {"torchvision": "ConvNextImageProcessor", "pil": "ConvNextImageProcessorPil"}),
+ ("cvt", {"torchvision": "ConvNextImageProcessor", "pil": "ConvNextImageProcessorPil"}),
+ ("data2vec-vision", {"torchvision": "BeitImageProcessor", "pil": "BeitImageProcessorPil"}),
+ ("deimv2", {"torchvision": "RTDetrImageProcessor", "pil": "RTDetrImageProcessorPil"}),
+ ("depth_anything", {"torchvision": "DPTImageProcessor", "pil": "DPTImageProcessorPil"}),
+ ("dinat", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("dinov2", {"torchvision": "BitImageProcessor", "pil": "BitImageProcessorPil"}),
+ ("donut-swin", {"torchvision": "DonutImageProcessor", "pil": "DonutImageProcessorPil"}),
+ ("edgetam", {"torchvision": "Sam2ImageProcessor"}),
+ ("emu3", {"pil": "Emu3ImageProcessor"}),
+ ("eomt_dinov3", {"torchvision": "EomtImageProcessor", "pil": "EomtImageProcessorPil"}),
+ ("exaone4_5", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("florence2", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("focalnet", {"torchvision": "BitImageProcessor", "pil": "BitImageProcessorPil"}),
+ ("gemma3n", {"torchvision": "SiglipImageProcessor", "pil": "SiglipImageProcessorPil"}),
+ ("git", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("granite4_vision", {"torchvision": "LlavaNextImageProcessor", "pil": "LlavaNextImageProcessorPil"}),
+ ("groupvit", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("hiera", {"torchvision": "BitImageProcessor", "pil": "BitImageProcessorPil"}),
+ ("ijepa", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("instructblip", {"torchvision": "BlipImageProcessor", "pil": "BlipImageProcessorPil"}),
+ ("internvl", {"torchvision": "GotOcr2ImageProcessor", "pil": "GotOcr2ImageProcessorPil"}),
+ ("kosmos-2", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("kosmos-2.5", {"torchvision": "Kosmos2_5ImageProcessor", "pil": "Kosmos2_5ImageProcessorPil"}),
+ ("layoutxlm", {"torchvision": "LayoutLMv2ImageProcessor", "pil": "LayoutLMv2ImageProcessorPil"}),
+ ("lighton_ocr", {"torchvision": "PixtralImageProcessor", "pil": "PixtralImageProcessorPil"}),
+ ("llava_next_video", {"torchvision": "LlavaNextImageProcessor", "pil": "LlavaNextImageProcessorPil"}),
+ ("lw_detr", {"torchvision": "DeformableDetrImageProcessor", "pil": "DeformableDetrImageProcessorPil"}),
+ ("metaclip_2", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("mgp-str", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("mistral3", {"torchvision": "PixtralImageProcessor", "pil": "PixtralImageProcessorPil"}),
+ ("mlcd", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ (
+ "mm-grounding-dino",
+ {
+ "torchvision": "GroundingDinoImageProcessor",
+ "pil": "GroundingDinoImageProcessorPil",
+ },
+ ),
+ ("mobilevitv2", {"torchvision": "MobileViTImageProcessor", "pil": "MobileViTImageProcessorPil"}),
+ ("omdet-turbo", {"torchvision": "DetrImageProcessor", "pil": "DetrImageProcessorPil"}),
+ ("paligemma", {"torchvision": "SiglipImageProcessor", "pil": "SiglipImageProcessorPil"}),
+ ("pixio", {"torchvision": "BitImageProcessor", "pil": "BitImageProcessorPil"}),
+ ("pp_ocrv5_mobile_det", {"torchvision": "PPOCRV5ServerDetImageProcessor"}),
+ ("pp_ocrv5_mobile_rec", {"torchvision": "PPOCRV5ServerRecImageProcessor"}),
+ ("pvt_v2", {"torchvision": "PvtImageProcessor", "pil": "PvtImageProcessorPil"}),
+ ("qianfan_ocr", {"torchvision": "GotOcr2ImageProcessor", "pil": "GotOcr2ImageProcessorPil"}),
+ ("qwen2_5_omni", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("qwen2_5_vl", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("qwen3_5", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("qwen3_5_moe", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("qwen3_omni_moe", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("qwen3_vl", {"torchvision": "Qwen2VLImageProcessor", "pil": "Qwen2VLImageProcessorPil"}),
+ ("regnet", {"torchvision": "ConvNextImageProcessor", "pil": "ConvNextImageProcessorPil"}),
+ ("resnet", {"torchvision": "ConvNextImageProcessor", "pil": "ConvNextImageProcessorPil"}),
+ ("sam2_video", {"torchvision": "Sam2ImageProcessor"}),
+ ("sam3_lite_text", {"torchvision": "Sam3ImageProcessor"}),
+ ("sam3_tracker", {"torchvision": "Sam3ImageProcessor"}),
+ ("sam3_tracker_video", {"torchvision": "Sam3ImageProcessor"}),
+ ("sam3_video", {"torchvision": "Sam3ImageProcessor"}),
+ ("sam_hq", {"torchvision": "SamImageProcessor", "pil": "SamImageProcessorPil"}),
+ ("shieldgemma2", {"torchvision": "Gemma3ImageProcessor", "pil": "Gemma3ImageProcessorPil"}),
+ ("slanet", {"torchvision": "SLANeXtImageProcessor"}),
+ ("swiftformer", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("swin", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("swinv2", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("t5gemma2", {"torchvision": "Gemma3ImageProcessor", "pil": "Gemma3ImageProcessorPil"}),
+ ("t5gemma2_encoder", {"torchvision": "Gemma3ImageProcessor", "pil": "Gemma3ImageProcessorPil"}),
+ ("table-transformer", {"torchvision": "DetrImageProcessor", "pil": "DetrImageProcessorPil"}),
+ ("timesformer", {"pil": "VideoMAEImageProcessorPil", "torchvision": "VideoMAEImageProcessor"}),
+ ("timm_wrapper", {"pil": "TimmWrapperImageProcessor"}),
+ ("trocr", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("udop", {"torchvision": "LayoutLMv3ImageProcessor", "pil": "LayoutLMv3ImageProcessorPil"}),
+ ("upernet", {"torchvision": "SegformerImageProcessor", "pil": "SegformerImageProcessorPil"}),
+ ("video_llava", {"pil": "VideoLlavaImageProcessor"}),
+ ("vipllava", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ("vit_mae", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("vit_msn", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
+ ("vivit", {"torchvision": "VivitImageProcessor"}),
+ ("xclip", {"torchvision": "CLIPImageProcessor", "pil": "CLIPImageProcessorPil"}),
+ ]
+ )
+
+ IMAGE_PROCESSOR_MAPPING_NAMES.update(MISSING_IMAGE_PROCESSOR_MAPPING_NAMES)
+
+IMAGE_PROCESSOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES)
+
+
+def get_image_processor_class_from_name(class_name: str):
+ """Resolve an image processor class name to its class. Handles both base names (e.g. CLIPImageProcessor)
+ and PIL backend names (e.g. CLIPImageProcessorPil). No recursion needed since names are direct."""
+ if class_name == "BaseImageProcessorFast":
+ # kept for backward compatibility - return TorchvisionBackend
+ from ...image_processing_backends import TorchvisionBackend
+
+ return TorchvisionBackend
+
+ # First, check registered extra content (user-registered classes)
+ for mapping in IMAGE_PROCESSOR_MAPPING._extra_content.values():
+ for extractor_class in mapping.values():
+ if isinstance(extractor_class, type) and getattr(extractor_class, "__name__", None) == class_name:
+ return extractor_class
+
+ # Check the mapping names - class names are either base (torchvision) or base+Pil (pil)
+ for model_type, extractors_dict in IMAGE_PROCESSOR_MAPPING_NAMES.items():
+ if class_name in extractors_dict.values():
+ module_name = model_type_to_module_name(model_type)
+ module = importlib.import_module(f".{module_name}", "transformers.models")
+ try:
+ return getattr(module, class_name)
+ except AttributeError:
+ continue
+
+ # Fallback: class may be in main init (e.g. when dep is missing, returns dummy)
+ main_module = importlib.import_module("transformers")
+ if hasattr(main_module, class_name):
+ return getattr(main_module, class_name)
+
+ return None
+
+
+def get_image_processor_config(
+ pretrained_model_name_or_path: str | os.PathLike,
+ cache_dir: str | os.PathLike | None = None,
+ force_download: bool = False,
+ proxies: dict[str, str] | None = None,
+ token: bool | str | None = None,
+ revision: str | None = None,
+ local_files_only: bool = False,
+ **kwargs,
+):
+ """
+ Loads the image processor configuration from a pretrained model image processor configuration.
+
+ Args:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a configuration file saved using the
+ [`~ProcessorMixin.save_pretrained`] method, e.g., `./my_model_directory/`.
+
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
+ cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the configuration files and override the cached versions if they
+ exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ local_files_only (`bool`, *optional*, defaults to `False`):
+ If `True`, will only try to load the image processor configuration from local files.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Returns:
+ `Dict`: The configuration of the image processor.
+
+ Examples:
+
+ ```python
+ # Download configuration from huggingface.co and cache.
+ image_processor_config = get_image_processor_config("google-bert/bert-base-uncased")
+ # This model does not have a image processor config so the result will be an empty dict.
+ image_processor_config = get_image_processor_config("FacebookAI/xlm-roberta-base")
+
+ # Save a pretrained image processor locally and you can reload its config
+ from transformers import AutoImageProcessor
+
+ image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k")
+ image_processor.save_pretrained("image-processor-test")
+ image_processor_config = get_image_processor_config("image-processor-test")
+ ```"""
+ # Load with a priority given to the nested processor config, if available in repo
+ resolved_processor_file = cached_file(
+ pretrained_model_name_or_path,
+ filename=PROCESSOR_NAME,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ )
+ resolved_image_processor_file = cached_file(
+ pretrained_model_name_or_path,
+ filename=IMAGE_PROCESSOR_NAME,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ )
+
+ # An empty list if none of the possible files is found in the repo
+ if not resolved_image_processor_file and not resolved_processor_file:
+ logger.info("Could not locate the image processor configuration file.")
+ return {}
+
+ # Load image_processor dict. Priority goes as (nested config if found -> image processor config)
+ # We are downloading both configs because almost all models have a `processor_config.json` but
+ # not all of these are nested. We need to check if it was saved recently as nested or if it is legacy style
+ image_processor_dict = {}
+ if resolved_processor_file is not None:
+ processor_dict = safe_load_json_file(resolved_processor_file)
+ if "image_processor" in processor_dict:
+ image_processor_dict = processor_dict["image_processor"]
+
+ if resolved_image_processor_file is not None and image_processor_dict is None:
+ image_processor_dict = safe_load_json_file(resolved_image_processor_file)
+
+ return image_processor_dict
+
+
+def _resolve_backend(backend: str | None, use_fast: bool | None, base_class_name: str | None) -> str:
+ """Resolve raw backend inputs to a concrete backend name ('torchvision' or 'pil').
+
+ Handles, in order:
+ - Deprecated ``use_fast`` flag: warns and converts to an explicit backend string when no
+ explicit backend is given.
+ - Explicit backend string: returned as-is.
+ - None resolution: forces 'pil' for processors in DEFAULT_TO_PIL_BACKEND_IMAGE_PROCESSORS
+ (Lanczos interpolation, unsupported by torchvision); otherwise picks 'torchvision' when
+ available, falling back to 'pil'.
+ """
+ if use_fast is not None:
+ logger.warning_once(
+ "The `use_fast` parameter is deprecated and will be removed in a future version. "
+ 'Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.'
+ )
+ if backend is None:
+ backend = "torchvision" if use_fast else "pil"
+
+ if backend is None:
+ if base_class_name in DEFAULT_TO_PIL_BACKEND_IMAGE_PROCESSORS:
+ return "pil"
+ return "torchvision" if is_torchvision_available() else "pil"
+
+ return backend
+
+
+def _load_class_with_fallback(mapping, backend):
+ """
+ Load an image processor class from a backend-to-class mapping, with fallback.
+
+ Tries the requested backend first, then the opposite standard backend,
+ then any remaining backends. Works with both string class names and resolved class objects.
+
+ Unavailable backends are detected via DummyObject: classes whose required libraries are missing
+ are represented as DummyObject subclasses (is_dummy=True). When the torchvision backend is
+ missing but a PIL variant exists, _LazyModule transparently returns the PIL class with its own
+ warning, so _load_class_with_fallback naturally receives a usable class without extra gating.
+
+ Args:
+ mapping: dict mapping backend names (str) to class names (str) or class objects (type).
+ backend: the preferred backend name (e.g. "torchvision", "pil").
+
+ Returns:
+ The loaded class, or None if no class could be loaded.
+ """
+ backends_to_try = [backend] + [k for k in mapping if k != backend]
+
+ for b in backends_to_try:
+ value = mapping.get(b)
+ if value is None:
+ continue
+
+ # Value can be a class object (from resolved mapping) or a string class name
+ if isinstance(value, type):
+ processor_class = value
+ else:
+ processor_class = get_image_processor_class_from_name(value)
+
+ if processor_class is None or getattr(processor_class, "is_dummy", False):
+ continue
+
+ if b != backend:
+ logger.warning_once(f"Requested {backend} backend is not available. Falling back to {b} backend.")
+ return processor_class
+
+ return None
+
+
+def _find_mapping_for_image_processor(base_class_name: str) -> dict | None:
+ """
+ Find the backend->class mapping that contains base_class_name in its values.
+ Returns the mapping dict (including any custom registered backends) or None.
+ """
+
+ def _value_matches(val, name: str) -> bool:
+ if val is None:
+ return False
+ if isinstance(val, str):
+ return val == name
+ if isinstance(val, type):
+ return getattr(val, "__name__", None) == name
+ return False
+
+ for mapping_dict in IMAGE_PROCESSOR_MAPPING_NAMES.values():
+ if any(_value_matches(v, base_class_name) for v in mapping_dict.values()):
+ return mapping_dict
+
+ for content in IMAGE_PROCESSOR_MAPPING._extra_content.values():
+ if any(_value_matches(v, base_class_name) for v in content.values()):
+ return content
+
+ return None
+
+
+def _load_backend_class(base_class_name, backend, is_legacy_fast=False):
+ """
+ Load image processor class for a given backend. Uses the mapping from
+ IMAGE_PROCESSOR_MAPPING when base_class_name is found in its values (so config
+ overrides and custom backends are respected). Falls back to base+Pil convention
+ for remote code / unknown processors.
+ """
+ mapping = _find_mapping_for_image_processor(base_class_name)
+ if mapping is None:
+ mapping = {
+ "torchvision": base_class_name,
+ "pil": base_class_name + "Pil",
+ }
+ processor_class = _load_class_with_fallback(mapping, backend)
+
+ # For legacy Fast classes, try the original Fast class name as last resort
+ if processor_class is None and is_legacy_fast:
+ processor_class = get_image_processor_class_from_name(base_class_name + "Fast")
+
+ return processor_class
+
+
+def _resolve_auto_map_class_ref(auto_map, backend):
+ """Extract the class reference string from an auto_map entry based on backend preference.
+
+ Returns:
+ A string that may be:
+ - A simple class name (e.g. `"MyImageProcessor"`)
+ - A Hub reference in the form `upstream_repo--path/to/file.py::ClassName`, where the part before
+ `--` is the upstream repo ID (used for trust_remote_code resolution).
+ """
+ if isinstance(auto_map, dict):
+ return auto_map.get(backend) or next(iter(auto_map.values()))
+ if isinstance(auto_map, (list, tuple)):
+ if backend == "torchvision" and len(auto_map) > 1 and auto_map[1] is not None:
+ return auto_map[1]
+ return auto_map[0]
+ # Single string (legacy)
+ return auto_map
+
+
+@requires(backends=("vision",))
+class AutoImageProcessor:
+ r"""
+ This is a generic image processor class that will be instantiated as one of the image processor classes of the
+ library when created with the [`AutoImageProcessor.from_pretrained`] class method.
+
+ This class cannot be instantiated directly using `__init__()` (throws an error).
+ """
+
+ def __init__(self):
+ raise OSError(
+ "AutoImageProcessor is designed to be instantiated "
+ "using the `AutoImageProcessor.from_pretrained(pretrained_model_name_or_path)` method."
+ )
+
+ @classmethod
+ @replace_list_option_in_docstrings(IMAGE_PROCESSOR_MAPPING_NAMES)
+ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
+ r"""
+ Instantiate one of the image processor classes of the library from a pretrained model vocabulary.
+
+ The image processor class to instantiate is selected based on the `model_type` property of the config object
+ (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's
+ missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
+
+ List options
+
+ Params:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained image_processor hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a image processor file saved using the
+ [`~image_processing_utils.ImageProcessingMixin.save_pretrained`] method, e.g.,
+ `./my_model_directory/`.
+ - a path to a saved image processor JSON *file*, e.g.,
+ `./my_model_directory/preprocessor_config.json`.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model image processor should be cached if the
+ standard cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the image processor files and override the cached versions if
+ they exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ use_fast (`bool`, *optional*, defaults to `False`):
+ **Deprecated**: Use `backend="torchvision"` instead. This parameter is kept for backward compatibility.
+ Use a fast torchvision-based image processor if it is supported for a given model.
+ If a fast image processor is not available for a given model, a normal numpy-based image processor
+ is returned instead.
+ backend (`str`, *optional*, defaults to `None`):
+ The backend to use for image processing. Can be:
+ - `None`: Automatically select the best available backend (torchvision if available, otherwise pil)
+ - `"torchvision"`: Use Torchvision backend (GPU-accelerated, faster)
+ - `"pil"`: Use PIL backend (portable, CPU-only)
+ - Any custom backend name registered via `register()` method
+ return_unused_kwargs (`bool`, *optional*, defaults to `False`):
+ If `False`, then this function returns just the final image processor object. If `True`, then this
+ functions returns a `Tuple(image_processor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not image processor attributes: i.e., the part of
+ `kwargs` which has not been used to update `image_processor` and is otherwise ignored.
+ trust_remote_code (`bool`, *optional*, defaults to `False`):
+ Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
+ image_processor_filename (`str`, *optional*, defaults to `"config.json"`):
+ The name of the file in the model directory to use for the image processor config.
+ kwargs (`dict[str, Any]`, *optional*):
+ The values in kwargs of any keys which are image processor attributes will be used to override the
+ loaded values. Behavior concerning key/value pairs whose keys are *not* image processor attributes is
+ controlled by the `return_unused_kwargs` keyword parameter.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoImageProcessor
+
+ >>> # Download image processor from huggingface.co and cache.
+ >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k")
+
+ >>> # If image processor files are in a directory (e.g. image processor was saved using *save_pretrained('./test/saved_model/')*)
+ >>> # image_processor = AutoImageProcessor.from_pretrained("./test/saved_model/")
+ ```"""
+ config = kwargs.pop("config", None)
+ use_fast = kwargs.pop("use_fast", None)
+ backend_kwarg = kwargs.pop("backend", None)
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
+ kwargs["_from_auto"] = True
+
+ # Resolve the image processor config filename
+ if "image_processor_filename" in kwargs:
+ image_processor_filename = kwargs.pop("image_processor_filename")
+ elif is_timm_local_checkpoint(pretrained_model_name_or_path):
+ image_processor_filename = CONFIG_NAME
+ else:
+ image_processor_filename = IMAGE_PROCESSOR_NAME
+
+ # Load the image processor config
+
+ try:
+ config_dict, _ = ImageProcessingMixin.get_image_processor_dict(
+ pretrained_model_name_or_path, image_processor_filename=image_processor_filename, **kwargs
+ )
+ except Exception as initial_exception:
+ # Fallback for Hub TimmWrapper checkpoints (image processing in config.json, not preprocessor_config.json)
+ try:
+ config_dict, _ = ImageProcessingMixin.get_image_processor_dict(
+ pretrained_model_name_or_path, image_processor_filename=CONFIG_NAME, **kwargs
+ )
+ except Exception:
+ raise initial_exception
+
+ if not is_timm_config_dict(config_dict):
+ raise initial_exception
+
+ image_processor_type = config_dict.get("image_processor_type", None)
+ image_processor_auto_map = None
+ if "AutoImageProcessor" in config_dict.get("auto_map", {}):
+ image_processor_auto_map = config_dict["auto_map"]["AutoImageProcessor"]
+
+ # Backward compat: infer from feature extractor config
+ if image_processor_type is None and image_processor_auto_map is None:
+ feature_extractor_class = config_dict.pop("feature_extractor_type", None)
+ if feature_extractor_class is not None:
+ image_processor_type = feature_extractor_class.replace("FeatureExtractor", "ImageProcessor")
+ if "AutoFeatureExtractor" in config_dict.get("auto_map", {}):
+ feature_extractor_auto_map = config_dict["auto_map"]["AutoFeatureExtractor"]
+ image_processor_auto_map = feature_extractor_auto_map.replace("FeatureExtractor", "ImageProcessor")
+
+ # If not in image processor config, try the model config (override image_processor_auto_map if trust_remote_code is False)
+ if image_processor_type is None and (image_processor_auto_map is None or trust_remote_code is False):
+ if not isinstance(config, PreTrainedConfig):
+ config = AutoConfig.from_pretrained(
+ pretrained_model_name_or_path,
+ trust_remote_code=trust_remote_code,
+ **kwargs,
+ )
+ image_processor_type = getattr(config, "image_processor_type", None)
+ if hasattr(config, "auto_map") and "AutoImageProcessor" in config.auto_map:
+ image_processor_auto_map = config.auto_map["AutoImageProcessor"]
+
+ # Derive base_class_name from image_processor_type
+ is_legacy_fast = False
+ base_class_name = None
+ if image_processor_type is not None:
+ is_legacy_fast = image_processor_type.endswith("Fast")
+ base_class_name = image_processor_type[:-4] if is_legacy_fast else image_processor_type
+
+ backend = _resolve_backend(backend_kwarg, use_fast, base_class_name)
+
+ image_processor_class = None
+ if base_class_name is not None:
+ image_processor_class = _load_backend_class(base_class_name, backend, is_legacy_fast)
+
+ # Handle remote code
+ has_remote_code = image_processor_auto_map is not None
+ has_local_code = image_processor_class is not None or type(config) in IMAGE_PROCESSOR_MAPPING
+ explicit_local_code = has_local_code and not (
+ image_processor_class or _load_class_with_fallback(IMAGE_PROCESSOR_MAPPING[type(config)], backend)
+ ).__module__.startswith("transformers.")
+ if has_remote_code:
+ class_ref = _resolve_auto_map_class_ref(image_processor_auto_map, backend)
+ upstream_repo = class_ref.split("--")[0] if "--" in class_ref else None
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
+ )
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ image_processor_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
+ _ = kwargs.pop("code_revision", None)
+ image_processor_class.register_for_auto_class()
+ return image_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+ elif image_processor_class is not None:
+ return image_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+ # Last try: we use the IMAGE_PROCESSOR_MAPPING.
+ elif type(config) in IMAGE_PROCESSOR_MAPPING:
+ image_processor_mapping = IMAGE_PROCESSOR_MAPPING[type(config)]
+ image_processor_class = _load_class_with_fallback(image_processor_mapping, backend)
+
+ if image_processor_class is not None:
+ return image_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ available = [k for k, v in image_processor_mapping.items() if v is not None]
+ raise ValueError(f"Could not find image processor class. Available backends: {', '.join(available)}")
+ raise ValueError(
+ f"Unrecognized image processor in {pretrained_model_name_or_path}. Should have a "
+ f"`image_processor_type` key in its {IMAGE_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following "
+ f"`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in IMAGE_PROCESSOR_MAPPING_NAMES)}"
+ )
+
+ @staticmethod
+ def register(
+ config_class,
+ slow_image_processor_class: type | None = None,
+ fast_image_processor_class: type | None = None,
+ image_processor_classes: dict[str, type] | None = None,
+ exist_ok: bool = False,
+ ):
+ """
+ Register a new image processor for this class.
+
+ Args:
+ config_class ([`PreTrainedConfig`]):
+ The configuration corresponding to the model to register.
+ slow_image_processor_class (`type`, *optional*):
+ The PIL backend image processor class (deprecated, use `image_processor_classes={"pil": ...}`).
+ fast_image_processor_class (`type`, *optional*):
+ The Torchvision backend image processor class (deprecated, use `image_processor_classes={"torchvision": ...}`).
+ image_processor_classes (`dict[str, type]`, *optional*):
+ Dictionary mapping backend names to image processor classes. Allows registering custom backends.
+ Example: `{"pil": MyPilProcessor, "torchvision": MyTorchvisionProcessor, "custom": MyCustomProcessor}`
+ exist_ok (`bool`, *optional*, defaults to `False`):
+ If `True`, allow overwriting existing registrations.
+ """
+ # Handle backward compatibility: convert old parameters to new format
+ if image_processor_classes is None:
+ image_processor_classes = {}
+ if slow_image_processor_class is not None:
+ image_processor_classes["pil"] = slow_image_processor_class
+ if fast_image_processor_class is not None:
+ image_processor_classes["torchvision"] = fast_image_processor_class
+
+ if not image_processor_classes:
+ raise ValueError(
+ "You need to specify at least one image processor class. "
+ "Use `image_processor_classes={'backend_name': ProcessorClass}` or the deprecated "
+ "`slow_image_processor_class`/`fast_image_processor_class` parameters."
+ )
+
+ # Avoid resetting existing processors if we are passing partial updates
+ if config_class in IMAGE_PROCESSOR_MAPPING._extra_content:
+ existing_mapping = IMAGE_PROCESSOR_MAPPING[config_class]
+ existing_mapping.update(image_processor_classes)
+ image_processor_classes = existing_mapping
+
+ # Validate that all classes are proper image processor classes
+ from ...image_processing_utils import BaseImageProcessor
+
+ for backend_key, processor_class in image_processor_classes.items():
+ if processor_class is not None and not issubclass(processor_class, BaseImageProcessor):
+ raise ValueError(
+ f"Image processor class for backend '{backend_key}' must inherit from `BaseImageProcessor`. "
+ f"Got: {processor_class}"
+ )
+ IMAGE_PROCESSOR_MAPPING.register(config_class, image_processor_classes, exist_ok=exist_ok)
+
+
+__all__ = ["IMAGE_PROCESSOR_MAPPING", "AutoImageProcessor"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/modeling_auto.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/modeling_auto.py
new file mode 100644
index 0000000000000000000000000000000000000000..2eec91869ce4d664b7e3d438a6d4379e7de63df5
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/modeling_auto.py
@@ -0,0 +1,2434 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Auto Model class."""
+
+import os
+from collections import OrderedDict
+from typing import TYPE_CHECKING
+
+from ...utils import logging
+from .auto_factory import (
+ _BaseAutoBackboneClass,
+ _BaseAutoModelClass,
+ _LazyAutoMapping,
+ auto_class_update,
+)
+from .configuration_auto import CONFIG_MAPPING_NAMES
+
+
+if TYPE_CHECKING:
+ from ...generation import GenerationMixin
+ from ...modeling_utils import PreTrainedModel
+
+ # class for better type annotations
+ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
+ pass
+
+
+logger = logging.get_logger(__name__)
+
+MODEL_MAPPING_NAMES = OrderedDict(
+ [
+ # Base model mapping
+ ("afmoe", "AfmoeModel"),
+ ("aimv2", "Aimv2Model"),
+ ("aimv2_vision_model", "Aimv2VisionModel"),
+ ("albert", "AlbertModel"),
+ ("align", "AlignModel"),
+ ("altclip", "AltCLIPModel"),
+ ("apertus", "ApertusModel"),
+ ("arcee", "ArceeModel"),
+ ("aria", "AriaModel"),
+ ("aria_text", "AriaTextModel"),
+ ("audio-spectrogram-transformer", "ASTModel"),
+ ("audioflamingo3", "AudioFlamingo3ForConditionalGeneration"),
+ ("audioflamingo3_encoder", "AudioFlamingo3Encoder"),
+ ("autoformer", "AutoformerModel"),
+ ("aya_vision", "AyaVisionModel"),
+ ("bamba", "BambaModel"),
+ ("bark", "BarkModel"),
+ ("bart", "BartModel"),
+ ("beit", "BeitModel"),
+ ("bert", "BertModel"),
+ ("bert-generation", "BertGenerationEncoder"),
+ ("big_bird", "BigBirdModel"),
+ ("bigbird_pegasus", "BigBirdPegasusModel"),
+ ("biogpt", "BioGptModel"),
+ ("bit", "BitModel"),
+ ("bitnet", "BitNetModel"),
+ ("blenderbot", "BlenderbotModel"),
+ ("blenderbot-small", "BlenderbotSmallModel"),
+ ("blip", "BlipModel"),
+ ("blip-2", "Blip2Model"),
+ ("blip_2_qformer", "Blip2QFormerModel"),
+ ("bloom", "BloomModel"),
+ ("blt", "BltModel"),
+ ("bridgetower", "BridgeTowerModel"),
+ ("bros", "BrosModel"),
+ ("camembert", "CamembertModel"),
+ ("canine", "CanineModel"),
+ ("chameleon", "ChameleonModel"),
+ ("chinese_clip", "ChineseCLIPModel"),
+ ("chinese_clip_vision_model", "ChineseCLIPVisionModel"),
+ ("clap", "ClapModel"),
+ ("clip", "CLIPModel"),
+ ("clip_text_model", "CLIPTextModel"),
+ ("clip_vision_model", "CLIPVisionModel"),
+ ("clipseg", "CLIPSegModel"),
+ ("clvp", "ClvpModelForConditionalGeneration"),
+ ("codegen", "CodeGenModel"),
+ ("cohere", "CohereModel"),
+ ("cohere2", "Cohere2Model"),
+ ("cohere2_moe", "Cohere2MoeModel"),
+ ("cohere2_vision", "Cohere2VisionModel"),
+ ("cohere_asr", "CohereAsrModel"),
+ ("conditional_detr", "ConditionalDetrModel"),
+ ("convbert", "ConvBertModel"),
+ ("convnext", "ConvNextModel"),
+ ("convnextv2", "ConvNextV2Model"),
+ ("cpmant", "CpmAntModel"),
+ ("csm", "CsmForConditionalGeneration"),
+ ("ctrl", "CTRLModel"),
+ ("cvt", "CvtModel"),
+ ("cwm", "CwmModel"),
+ ("d_fine", "DFineModel"),
+ ("dab-detr", "DabDetrModel"),
+ ("dac", "DacModel"),
+ ("data2vec-audio", "Data2VecAudioModel"),
+ ("data2vec-text", "Data2VecTextModel"),
+ ("data2vec-vision", "Data2VecVisionModel"),
+ ("dbrx", "DbrxModel"),
+ ("deberta", "DebertaModel"),
+ ("deberta-v2", "DebertaV2Model"),
+ ("decision_transformer", "DecisionTransformerModel"),
+ ("deepseek_v2", "DeepseekV2Model"),
+ ("deepseek_v3", "DeepseekV3Model"),
+ ("deepseek_v4", "DeepseekV4Model"),
+ ("deepseek_vl", "DeepseekVLModel"),
+ ("deepseek_vl_hybrid", "DeepseekVLHybridModel"),
+ ("deformable_detr", "DeformableDetrModel"),
+ ("deimv2", "Deimv2Model"),
+ ("deit", "DeiTModel"),
+ ("depth_pro", "DepthProModel"),
+ ("detr", "DetrModel"),
+ ("dia", "DiaModel"),
+ ("diffllama", "DiffLlamaModel"),
+ ("dinat", "DinatModel"),
+ ("dinov2", "Dinov2Model"),
+ ("dinov2_with_registers", "Dinov2WithRegistersModel"),
+ ("dinov3_convnext", "DINOv3ConvNextModel"),
+ ("dinov3_vit", "DINOv3ViTModel"),
+ ("distilbert", "DistilBertModel"),
+ ("doge", "DogeModel"),
+ ("donut-swin", "DonutSwinModel"),
+ ("dots1", "Dots1Model"),
+ ("dpr", "DPRQuestionEncoder"),
+ ("dpt", "DPTModel"),
+ ("edgetam", "EdgeTamModel"),
+ ("edgetam_video", "EdgeTamVideoModel"),
+ ("edgetam_vision_model", "EdgeTamVisionModel"),
+ ("efficientloftr", "EfficientLoFTRModel"),
+ ("efficientnet", "EfficientNetModel"),
+ ("electra", "ElectraModel"),
+ ("emu3", "Emu3Model"),
+ ("encodec", "EncodecModel"),
+ ("ernie", "ErnieModel"),
+ ("ernie4_5", "Ernie4_5Model"),
+ ("ernie4_5_moe", "Ernie4_5_MoeModel"),
+ ("ernie4_5_vl_moe", "Ernie4_5_VLMoeModel"),
+ ("esm", "EsmModel"),
+ ("eurobert", "EuroBertModel"),
+ ("evolla", "EvollaModel"),
+ ("exaone4", "Exaone4Model"),
+ ("exaone4_5", "Exaone4_5_Model"),
+ ("exaone4_5_vision", "Exaone4_5_VisionModel"),
+ ("exaone_moe", "ExaoneMoeModel"),
+ ("falcon", "FalconModel"),
+ ("falcon_h1", "FalconH1Model"),
+ ("falcon_mamba", "FalconMambaModel"),
+ ("fast_vlm", "FastVlmModel"),
+ ("fastspeech2_conformer", "FastSpeech2ConformerModel"),
+ ("fastspeech2_conformer_with_hifigan", "FastSpeech2ConformerWithHifiGan"),
+ ("flaubert", "FlaubertModel"),
+ ("flava", "FlavaModel"),
+ ("flex_olmo", "FlexOlmoModel"),
+ ("florence2", "Florence2Model"),
+ ("fnet", "FNetModel"),
+ ("focalnet", "FocalNetModel"),
+ ("fsmt", "FSMTModel"),
+ ("funnel", ("FunnelModel", "FunnelBaseModel")),
+ ("fuyu", "FuyuModel"),
+ ("gemma", "GemmaModel"),
+ ("gemma2", "Gemma2Model"),
+ ("gemma3", "Gemma3Model"),
+ ("gemma3_text", "Gemma3TextModel"),
+ ("gemma3n", "Gemma3nModel"),
+ ("gemma3n_audio", "Gemma3nAudioEncoder"),
+ ("gemma3n_text", "Gemma3nTextModel"),
+ ("gemma3n_vision", "TimmWrapperModel"),
+ ("gemma4", "Gemma4Model"),
+ ("gemma4_audio", "Gemma4AudioModel"),
+ ("gemma4_text", "Gemma4TextModel"),
+ ("gemma4_vision", "Gemma4VisionModel"),
+ ("git", "GitModel"),
+ ("glm", "GlmModel"),
+ ("glm4", "Glm4Model"),
+ ("glm46v", "Glm46VModel"),
+ ("glm4_moe", "Glm4MoeModel"),
+ ("glm4_moe_lite", "Glm4MoeLiteModel"),
+ ("glm4v", "Glm4vModel"),
+ ("glm4v_moe", "Glm4vMoeModel"),
+ ("glm4v_moe_text", "Glm4vMoeTextModel"),
+ ("glm4v_moe_vision", "Glm4vMoeVisionModel"),
+ ("glm4v_text", "Glm4vTextModel"),
+ ("glm4v_vision", "Glm4vVisionModel"),
+ ("glm_image", "GlmImageModel"),
+ ("glm_image_text", "GlmImageTextModel"),
+ ("glm_image_vision", "GlmImageVisionModel"),
+ ("glm_image_vqmodel", "GlmImageVQVAE"),
+ ("glm_moe_dsa", "GlmMoeDsaModel"),
+ ("glm_ocr", "GlmOcrModel"),
+ ("glm_ocr_text", "GlmOcrTextModel"),
+ ("glm_ocr_vision", "GlmOcrVisionModel"),
+ ("glmasr", "GlmAsrForConditionalGeneration"),
+ ("glmasr_encoder", "GlmAsrEncoder"),
+ ("glpn", "GLPNModel"),
+ ("got_ocr2", "GotOcr2Model"),
+ ("gpt-sw3", "GPT2Model"),
+ ("gpt2", "GPT2Model"),
+ ("gpt_bigcode", "GPTBigCodeModel"),
+ ("gpt_neo", "GPTNeoModel"),
+ ("gpt_neox", "GPTNeoXModel"),
+ ("gpt_neox_japanese", "GPTNeoXJapaneseModel"),
+ ("gpt_oss", "GptOssModel"),
+ ("gptj", "GPTJModel"),
+ ("granite", "GraniteModel"),
+ ("granite4_vision", "Granite4VisionModel"),
+ ("granite_speech", "GraniteSpeechForConditionalGeneration"),
+ ("granitemoe", "GraniteMoeModel"),
+ ("granitemoehybrid", "GraniteMoeHybridModel"),
+ ("granitemoeshared", "GraniteMoeSharedModel"),
+ ("grounding-dino", "GroundingDinoModel"),
+ ("groupvit", "GroupViTModel"),
+ ("helium", "HeliumModel"),
+ ("hgnet_v2", "HGNetV2Backbone"),
+ ("hiera", "HieraModel"),
+ ("higgs_audio_v2", "HiggsAudioV2ForConditionalGeneration"),
+ ("higgs_audio_v2_tokenizer", "HiggsAudioV2TokenizerModel"),
+ ("hrm_text", "HrmTextModel"),
+ ("hubert", "HubertModel"),
+ ("hunyuan_v1_dense", "HunYuanDenseV1Model"),
+ ("hunyuan_v1_moe", "HunYuanMoEV1Model"),
+ ("hy_v3", "HYV3Model"),
+ ("hyperclovax", "HyperCLOVAXModel"),
+ ("ibert", "IBertModel"),
+ ("idefics", "IdeficsModel"),
+ ("idefics2", "Idefics2Model"),
+ ("idefics3", "Idefics3Model"),
+ ("idefics3_vision", "Idefics3VisionTransformer"),
+ ("ijepa", "IJepaModel"),
+ ("imagegpt", "ImageGPTModel"),
+ ("informer", "InformerModel"),
+ ("instructblip", "InstructBlipModel"),
+ ("instructblipvideo", "InstructBlipVideoModel"),
+ ("internvl", "InternVLModel"),
+ ("internvl_vision", "InternVLVisionModel"),
+ ("jais2", "Jais2Model"),
+ ("jamba", "JambaModel"),
+ ("janus", "JanusModel"),
+ ("jetmoe", "JetMoeModel"),
+ ("jina_embeddings_v3", "JinaEmbeddingsV3Model"),
+ ("kosmos-2", "Kosmos2Model"),
+ ("kosmos-2.5", "Kosmos2_5Model"),
+ ("kyutai_speech_to_text", "KyutaiSpeechToTextModel"),
+ ("laguna", "LagunaModel"),
+ ("lasr_ctc", "LasrForCTC"),
+ ("lasr_encoder", "LasrEncoder"),
+ ("layoutlm", "LayoutLMModel"),
+ ("layoutlmv2", "LayoutLMv2Model"),
+ ("layoutlmv3", "LayoutLMv3Model"),
+ ("led", "LEDModel"),
+ ("levit", "LevitModel"),
+ ("lfm2", "Lfm2Model"),
+ ("lfm2_moe", "Lfm2MoeModel"),
+ ("lfm2_vl", "Lfm2VlModel"),
+ ("lightglue", "LightGlueForKeypointMatching"),
+ ("lighton_ocr", "LightOnOcrModel"),
+ ("lilt", "LiltModel"),
+ ("llama", "LlamaModel"),
+ ("llama4", "Llama4ForConditionalGeneration"),
+ ("llama4_text", "Llama4TextModel"),
+ ("llava", "LlavaModel"),
+ ("llava_next", "LlavaNextModel"),
+ ("llava_next_video", "LlavaNextVideoModel"),
+ ("llava_onevision", "LlavaOnevisionModel"),
+ ("longcat_flash", "LongcatFlashModel"),
+ ("longformer", "LongformerModel"),
+ ("longt5", "LongT5Model"),
+ ("luke", "LukeModel"),
+ ("lw_detr", "LwDetrModel"),
+ ("lxmert", "LxmertModel"),
+ ("m2m_100", "M2M100Model"),
+ ("mamba", "MambaModel"),
+ ("mamba2", "Mamba2Model"),
+ ("marian", "MarianModel"),
+ ("markuplm", "MarkupLMModel"),
+ ("mask2former", "Mask2FormerModel"),
+ ("maskformer", "MaskFormerModel"),
+ ("maskformer-swin", "MaskFormerSwinModel"),
+ ("mbart", "MBartModel"),
+ ("megatron-bert", "MegatronBertModel"),
+ ("metaclip_2", "MetaClip2Model"),
+ ("mgp-str", "MgpstrForSceneTextRecognition"),
+ ("mimi", "MimiModel"),
+ ("minicpmv4_6", "MiniCPMV4_6Model"),
+ ("minimax", "MiniMaxModel"),
+ ("minimax_m2", "MiniMaxM2Model"),
+ ("ministral", "MinistralModel"),
+ ("ministral3", "Ministral3Model"),
+ ("mistral", "MistralModel"),
+ ("mistral3", "Mistral3Model"),
+ ("mistral4", "Mistral4Model"),
+ ("mixtral", "MixtralModel"),
+ ("mlcd", "MLCDVisionModel"), # Keep this to make some original hub repositories (from `DeepGlint-AI`) works
+ ("mlcd_vision_model", "MLCDVisionModel"),
+ ("mllama", "MllamaModel"),
+ ("mm-grounding-dino", "MMGroundingDinoModel"),
+ ("mobilebert", "MobileBertModel"),
+ ("mobilenet_v1", "MobileNetV1Model"),
+ ("mobilenet_v2", "MobileNetV2Model"),
+ ("mobilevit", "MobileViTModel"),
+ ("mobilevitv2", "MobileViTV2Model"),
+ ("modernbert", "ModernBertModel"),
+ ("modernbert-decoder", "ModernBertDecoderModel"),
+ ("modernvbert", "ModernVBertModel"),
+ ("moonshine", "MoonshineModel"),
+ ("moonshine_streaming", "MoonshineStreamingModel"),
+ ("moshi", "MoshiModel"),
+ ("mpnet", "MPNetModel"),
+ ("mpt", "MptModel"),
+ ("mra", "MraModel"),
+ ("mt5", "MT5Model"),
+ ("musicflamingo", "MusicFlamingoForConditionalGeneration"),
+ ("musicgen", "MusicgenModel"),
+ ("musicgen_melody", "MusicgenMelodyModel"),
+ ("mvp", "MvpModel"),
+ ("nanochat", "NanoChatModel"),
+ ("nemotron", "NemotronModel"),
+ ("nemotron_h", "NemotronHModel"),
+ ("nllb-moe", "NllbMoeModel"),
+ ("nomic_bert", "NomicBertModel"),
+ ("nystromformer", "NystromformerModel"),
+ ("olmo", "OlmoModel"),
+ ("olmo2", "Olmo2Model"),
+ ("olmo3", "Olmo3Model"),
+ ("olmo_hybrid", "OlmoHybridModel"),
+ ("olmoe", "OlmoeModel"),
+ ("omdet-turbo", "OmDetTurboForObjectDetection"),
+ ("oneformer", "OneFormerModel"),
+ ("openai-gpt", "OpenAIGPTModel"),
+ ("openai_privacy_filter", "OpenAIPrivacyFilterModel"),
+ ("opt", "OPTModel"),
+ ("ovis2", "Ovis2Model"),
+ ("owlv2", "Owlv2Model"),
+ ("owlvit", "OwlViTModel"),
+ ("paligemma", "PaliGemmaModel"),
+ ("parakeet_ctc", "ParakeetForCTC"),
+ ("parakeet_encoder", "ParakeetEncoder"),
+ ("parakeet_tdt", "ParakeetForTDT"),
+ ("patchtsmixer", "PatchTSMixerModel"),
+ ("patchtst", "PatchTSTModel"),
+ ("pe_audio", "PeAudioModel"),
+ ("pe_audio_encoder", "PeAudioEncoder"),
+ ("pe_audio_video", "PeAudioVideoModel"),
+ ("pe_audio_video_encoder", "PeAudioVideoEncoder"),
+ ("pe_video", "PeVideoModel"),
+ ("pe_video_encoder", "PeVideoEncoder"),
+ ("pegasus", "PegasusModel"),
+ ("pegasus_x", "PegasusXModel"),
+ ("perceiver", "PerceiverModel"),
+ ("perception_lm", "PerceptionLMModel"),
+ ("persimmon", "PersimmonModel"),
+ ("phi", "PhiModel"),
+ ("phi3", "Phi3Model"),
+ ("phi4_multimodal", "Phi4MultimodalModel"),
+ ("phimoe", "PhimoeModel"),
+ ("pi0", "PI0Model"),
+ ("pixio", "PixioModel"),
+ ("pixtral", "PixtralVisionModel"),
+ ("plbart", "PLBartModel"),
+ ("poolformer", "PoolFormerModel"),
+ ("pp_doclayout_v3", "PPDocLayoutV3Model"),
+ ("pp_ocrv5_mobile_rec", "PPOCRV5MobileRecModel"),
+ ("pp_ocrv5_server_rec", "PPOCRV5ServerRecModel"),
+ ("prophetnet", "ProphetNetModel"),
+ ("pvt", "PvtModel"),
+ ("pvt_v2", "PvtV2Model"),
+ ("qianfan_ocr", "QianfanOCRModel"),
+ ("qianfan_ocr_vision", "QianfanOCRVisionModel"),
+ ("qwen2", "Qwen2Model"),
+ ("qwen2_5_vl", "Qwen2_5_VLModel"),
+ ("qwen2_5_vl_text", "Qwen2_5_VLTextModel"),
+ ("qwen2_audio_encoder", "Qwen2AudioEncoder"),
+ ("qwen2_moe", "Qwen2MoeModel"),
+ ("qwen2_vl", "Qwen2VLModel"),
+ ("qwen2_vl_text", "Qwen2VLTextModel"),
+ ("qwen3", "Qwen3Model"),
+ ("qwen3_5", "Qwen3_5Model"),
+ ("qwen3_5_moe", "Qwen3_5MoeModel"),
+ ("qwen3_5_moe_text", "Qwen3_5MoeTextModel"),
+ ("qwen3_5_text", "Qwen3_5TextModel"),
+ ("qwen3_moe", "Qwen3MoeModel"),
+ ("qwen3_next", "Qwen3NextModel"),
+ ("qwen3_vl", "Qwen3VLModel"),
+ ("qwen3_vl_moe", "Qwen3VLMoeModel"),
+ ("qwen3_vl_moe_text", "Qwen3VLMoeTextModel"),
+ ("qwen3_vl_text", "Qwen3VLTextModel"),
+ ("recurrent_gemma", "RecurrentGemmaModel"),
+ ("reformer", "ReformerModel"),
+ ("regnet", "RegNetModel"),
+ ("rembert", "RemBertModel"),
+ ("resnet", "ResNetModel"),
+ ("rf_detr", "RfDetrModel"),
+ ("roberta", "RobertaModel"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormModel"),
+ ("roc_bert", "RoCBertModel"),
+ ("roformer", "RoFormerModel"),
+ ("rt_detr", "RTDetrModel"),
+ ("rt_detr_v2", "RTDetrV2Model"),
+ ("rwkv", "RwkvModel"),
+ ("sam", "SamModel"),
+ ("sam2", "Sam2Model"),
+ ("sam2_hiera_det_model", "Sam2HieraDetModel"),
+ ("sam2_video", "Sam2VideoModel"),
+ ("sam2_vision_model", "Sam2VisionModel"),
+ ("sam3", "Sam3Model"),
+ ("sam3_lite_text", "Sam3LiteTextModel"),
+ ("sam3_lite_text_text_model", "Sam3LiteTextTextModel"),
+ ("sam3_tracker", "Sam3TrackerModel"),
+ ("sam3_tracker", "Sam3TrackerModel"),
+ ("sam3_tracker_video", "Sam3TrackerVideoModel"),
+ ("sam3_video", "Sam3VideoModel"),
+ ("sam3_vision_model", "Sam3VisionModel"),
+ ("sam3_vit_model", "Sam3ViTModel"),
+ ("sam_hq", "SamHQModel"),
+ ("sam_hq_vision_model", "SamHQVisionModel"),
+ ("sam_vision_model", "SamVisionModel"),
+ ("seamless_m4t", "SeamlessM4TModel"),
+ ("seamless_m4t_v2", "SeamlessM4Tv2Model"),
+ ("seed_oss", "SeedOssModel"),
+ ("segformer", "SegformerModel"),
+ ("seggpt", "SegGptModel"),
+ ("sew", "SEWModel"),
+ ("sew-d", "SEWDModel"),
+ ("siglip", "SiglipModel"),
+ ("siglip2", "Siglip2Model"),
+ ("siglip2_vision_model", "Siglip2VisionModel"),
+ ("siglip_vision_model", "SiglipVisionModel"),
+ ("smollm3", "SmolLM3Model"),
+ ("smolvlm", "SmolVLMModel"),
+ ("smolvlm_vision", "SmolVLMVisionTransformer"),
+ ("solar_open", "SolarOpenModel"),
+ ("speech_to_text", "Speech2TextModel"),
+ ("speecht5", "SpeechT5Model"),
+ ("splinter", "SplinterModel"),
+ ("squeezebert", "SqueezeBertModel"),
+ ("stablelm", "StableLmModel"),
+ ("starcoder2", "Starcoder2Model"),
+ ("swiftformer", "SwiftFormerModel"),
+ ("swin", "SwinModel"),
+ ("swin2sr", "Swin2SRModel"),
+ ("swinv2", "Swinv2Model"),
+ ("switch_transformers", "SwitchTransformersModel"),
+ ("t5", "T5Model"),
+ ("t5gemma", "T5GemmaModel"),
+ ("t5gemma2", "T5Gemma2Model"),
+ ("t5gemma2_encoder", "T5Gemma2Encoder"),
+ ("table-transformer", "TableTransformerModel"),
+ ("tapas", "TapasModel"),
+ ("textnet", "TextNetModel"),
+ ("time_series_transformer", "TimeSeriesTransformerModel"),
+ ("timesfm", "TimesFmModel"),
+ ("timesfm2_5", "TimesFm2_5Model"),
+ ("timesformer", "TimesformerModel"),
+ ("timm_backbone", "TimmBackbone"),
+ ("timm_wrapper", "TimmWrapperModel"),
+ ("tvp", "TvpModel"),
+ ("udop", "UdopModel"),
+ ("umt5", "UMT5Model"),
+ ("unispeech", "UniSpeechModel"),
+ ("unispeech-sat", "UniSpeechSatModel"),
+ ("univnet", "UnivNetModel"),
+ ("uvdoc", "UVDocModel"),
+ ("vaultgemma", "VaultGemmaModel"),
+ ("vibevoice_acoustic_tokenizer", "VibeVoiceAcousticTokenizerModel"),
+ ("vibevoice_acoustic_tokenizer_decoder", "VibeVoiceAcousticTokenizerDecoderModel"),
+ ("vibevoice_acoustic_tokenizer_encoder", "VibeVoiceAcousticTokenizerEncoderModel"),
+ ("vibevoice_asr", "VibeVoiceAsrForConditionalGeneration"),
+ ("video_llama_3", "VideoLlama3Model"),
+ ("video_llama_3_vision", "VideoLlama3VisionModel"),
+ ("video_llava", "VideoLlavaModel"),
+ ("videomae", "VideoMAEModel"),
+ ("vilt", "ViltModel"),
+ ("vipllava", "VipLlavaModel"),
+ ("vision-text-dual-encoder", "VisionTextDualEncoderModel"),
+ ("visual_bert", "VisualBertModel"),
+ ("vit", "ViTModel"),
+ ("vit_mae", "ViTMAEModel"),
+ ("vit_msn", "ViTMSNModel"),
+ ("vitdet", "VitDetModel"),
+ ("vits", "VitsModel"),
+ ("vivit", "VivitModel"),
+ ("vjepa2", "VJEPA2Model"),
+ ("voxtral", "VoxtralForConditionalGeneration"),
+ ("voxtral_encoder", "VoxtralEncoder"),
+ ("voxtral_realtime", "VoxtralRealtimeForConditionalGeneration"),
+ ("voxtral_realtime_encoder", "VoxtralRealtimeEncoder"),
+ ("voxtral_realtime_text", "VoxtralRealtimeTextModel"),
+ ("wav2vec2", "Wav2Vec2Model"),
+ ("wav2vec2-bert", "Wav2Vec2BertModel"),
+ ("wav2vec2-conformer", "Wav2Vec2ConformerModel"),
+ ("wavlm", "WavLMModel"),
+ ("whisper", "WhisperModel"),
+ ("xclip", "XCLIPModel"),
+ ("xcodec", "XcodecModel"),
+ ("xglm", "XGLMModel"),
+ ("xlm", "XLMModel"),
+ ("xlm-roberta", "XLMRobertaModel"),
+ ("xlm-roberta-xl", "XLMRobertaXLModel"),
+ ("xlnet", "XLNetModel"),
+ ("xlstm", "xLSTMModel"),
+ ("xmod", "XmodModel"),
+ ("yolos", "YolosModel"),
+ ("yoso", "YosoModel"),
+ ("youtu", "YoutuModel"),
+ ("zamba", "ZambaModel"),
+ ("zamba2", "Zamba2Model"),
+ ]
+)
+
+MODEL_FOR_PRETRAINING_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for pre-training mapping
+ ("albert", "AlbertForPreTraining"),
+ ("audioflamingo3", "AudioFlamingo3ForConditionalGeneration"),
+ ("bart", "BartForConditionalGeneration"),
+ ("bert", "BertForPreTraining"),
+ ("big_bird", "BigBirdForPreTraining"),
+ ("bloom", "BloomForCausalLM"),
+ ("camembert", "CamembertForMaskedLM"),
+ ("colmodernvbert", "ColModernVBertForRetrieval"),
+ ("colpali", "ColPaliForRetrieval"),
+ ("colqwen2", "ColQwen2ForRetrieval"),
+ ("ctrl", "CTRLLMHeadModel"),
+ ("data2vec-text", "Data2VecTextForMaskedLM"),
+ ("deberta", "DebertaForMaskedLM"),
+ ("deberta-v2", "DebertaV2ForMaskedLM"),
+ ("distilbert", "DistilBertForMaskedLM"),
+ ("electra", "ElectraForPreTraining"),
+ ("ernie", "ErnieForPreTraining"),
+ ("evolla", "EvollaForProteinText2Text"),
+ ("exaone4", "Exaone4ForCausalLM"),
+ ("exaone_moe", "ExaoneMoeForCausalLM"),
+ ("falcon_mamba", "FalconMambaForCausalLM"),
+ ("flaubert", "FlaubertWithLMHeadModel"),
+ ("flava", "FlavaForPreTraining"),
+ ("florence2", "Florence2ForConditionalGeneration"),
+ ("fnet", "FNetForPreTraining"),
+ ("fsmt", "FSMTForConditionalGeneration"),
+ ("funnel", "FunnelForPreTraining"),
+ ("gemma3", "Gemma3ForConditionalGeneration"),
+ ("gemma4", "Gemma4ForConditionalGeneration"),
+ ("glmasr", "GlmAsrForConditionalGeneration"),
+ ("gpt-sw3", "GPT2LMHeadModel"),
+ ("gpt2", "GPT2LMHeadModel"),
+ ("gpt_bigcode", "GPTBigCodeForCausalLM"),
+ ("hiera", "HieraForPreTraining"),
+ ("ibert", "IBertForMaskedLM"),
+ ("idefics", "IdeficsForVisionText2Text"),
+ ("idefics2", "Idefics2ForConditionalGeneration"),
+ ("idefics3", "Idefics3ForConditionalGeneration"),
+ ("janus", "JanusForConditionalGeneration"),
+ ("layoutlm", "LayoutLMForMaskedLM"),
+ ("llava", "LlavaForConditionalGeneration"),
+ ("llava_next", "LlavaNextForConditionalGeneration"),
+ ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
+ ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
+ ("longformer", "LongformerForMaskedLM"),
+ ("luke", "LukeForMaskedLM"),
+ ("lxmert", "LxmertForPreTraining"),
+ ("mamba", "MambaForCausalLM"),
+ ("mamba2", "Mamba2ForCausalLM"),
+ ("megatron-bert", "MegatronBertForPreTraining"),
+ ("mistral3", "Mistral3ForConditionalGeneration"),
+ ("mistral4", "Mistral4ForCausalLM"),
+ ("mllama", "MllamaForConditionalGeneration"),
+ ("mobilebert", "MobileBertForPreTraining"),
+ ("mpnet", "MPNetForMaskedLM"),
+ ("mpt", "MptForCausalLM"),
+ ("mra", "MraForMaskedLM"),
+ ("musicflamingo", "MusicFlamingoForConditionalGeneration"),
+ ("mvp", "MvpForConditionalGeneration"),
+ ("nanochat", "NanoChatForCausalLM"),
+ ("nllb-moe", "NllbMoeForConditionalGeneration"),
+ ("openai-gpt", "OpenAIGPTLMHeadModel"),
+ ("paligemma", "PaliGemmaForConditionalGeneration"),
+ ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),
+ ("roberta", "RobertaForMaskedLM"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormForMaskedLM"),
+ ("roc_bert", "RoCBertForPreTraining"),
+ ("rwkv", "RwkvForCausalLM"),
+ ("splinter", "SplinterForPreTraining"),
+ ("squeezebert", "SqueezeBertForMaskedLM"),
+ ("switch_transformers", "SwitchTransformersForConditionalGeneration"),
+ ("t5", "T5ForConditionalGeneration"),
+ ("t5gemma", "T5GemmaForConditionalGeneration"),
+ ("t5gemma2", "T5Gemma2ForConditionalGeneration"),
+ ("tapas", "TapasForMaskedLM"),
+ ("unispeech", "UniSpeechForPreTraining"),
+ ("unispeech-sat", "UniSpeechSatForPreTraining"),
+ ("vibevoice_asr", "VibeVoiceAsrForConditionalGeneration"),
+ ("video_llava", "VideoLlavaForConditionalGeneration"),
+ ("videomae", "VideoMAEForPreTraining"),
+ ("vipllava", "VipLlavaForConditionalGeneration"),
+ ("visual_bert", "VisualBertForPreTraining"),
+ ("vit_mae", "ViTMAEForPreTraining"),
+ ("voxtral", "VoxtralForConditionalGeneration"),
+ ("voxtral_realtime", "VoxtralRealtimeForConditionalGeneration"),
+ ("wav2vec2", "Wav2Vec2ForPreTraining"),
+ ("wav2vec2-conformer", "Wav2Vec2ConformerForPreTraining"),
+ ("xlm", "XLMWithLMHeadModel"),
+ ("xlm-roberta", "XLMRobertaForMaskedLM"),
+ ("xlm-roberta-xl", "XLMRobertaXLForMaskedLM"),
+ ("xlnet", "XLNetLMHeadModel"),
+ ("xlstm", "xLSTMForCausalLM"),
+ ("xmod", "XmodForMaskedLM"),
+ ]
+)
+
+MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Causal LM mapping
+ ("afmoe", "AfmoeForCausalLM"),
+ ("apertus", "ApertusForCausalLM"),
+ ("arcee", "ArceeForCausalLM"),
+ ("aria_text", "AriaTextForCausalLM"),
+ ("bamba", "BambaForCausalLM"),
+ ("bart", "BartForCausalLM"),
+ ("bert", "BertLMHeadModel"),
+ ("bert-generation", "BertGenerationDecoder"),
+ ("big_bird", "BigBirdForCausalLM"),
+ ("bigbird_pegasus", "BigBirdPegasusForCausalLM"),
+ ("biogpt", "BioGptForCausalLM"),
+ ("bitnet", "BitNetForCausalLM"),
+ ("blenderbot", "BlenderbotForCausalLM"),
+ ("blenderbot-small", "BlenderbotSmallForCausalLM"),
+ ("bloom", "BloomForCausalLM"),
+ ("blt", "BltForCausalLM"),
+ ("camembert", "CamembertForCausalLM"),
+ ("codegen", "CodeGenForCausalLM"),
+ ("cohere", "CohereForCausalLM"),
+ ("cohere2", "Cohere2ForCausalLM"),
+ ("cohere2_moe", "Cohere2MoeForCausalLM"),
+ ("cpmant", "CpmAntForCausalLM"),
+ ("ctrl", "CTRLLMHeadModel"),
+ ("cwm", "CwmForCausalLM"),
+ ("data2vec-text", "Data2VecTextForCausalLM"),
+ ("dbrx", "DbrxForCausalLM"),
+ ("deepseek_v2", "DeepseekV2ForCausalLM"),
+ ("deepseek_v3", "DeepseekV3ForCausalLM"),
+ ("deepseek_v4", "DeepseekV4ForCausalLM"),
+ ("diffllama", "DiffLlamaForCausalLM"),
+ ("doge", "DogeForCausalLM"),
+ ("dots1", "Dots1ForCausalLM"),
+ ("electra", "ElectraForCausalLM"),
+ ("emu3", "Emu3ForCausalLM"),
+ ("ernie", "ErnieForCausalLM"),
+ ("ernie4_5", "Ernie4_5ForCausalLM"),
+ ("ernie4_5_moe", "Ernie4_5_MoeForCausalLM"),
+ ("exaone4", "Exaone4ForCausalLM"),
+ ("exaone_moe", "ExaoneMoeForCausalLM"),
+ ("falcon", "FalconForCausalLM"),
+ ("falcon_h1", "FalconH1ForCausalLM"),
+ ("falcon_mamba", "FalconMambaForCausalLM"),
+ ("flex_olmo", "FlexOlmoForCausalLM"),
+ ("fuyu", "FuyuForCausalLM"),
+ ("gemma", "GemmaForCausalLM"),
+ ("gemma2", "Gemma2ForCausalLM"),
+ ("gemma3", "Gemma3ForConditionalGeneration"),
+ ("gemma3_text", "Gemma3ForCausalLM"),
+ ("gemma3n", "Gemma3nForConditionalGeneration"),
+ ("gemma3n_text", "Gemma3nForCausalLM"),
+ ("gemma4", "Gemma4ForConditionalGeneration"),
+ ("gemma4_assistant", "Gemma4AssistantForCausalLM"),
+ ("gemma4_text", "Gemma4ForCausalLM"),
+ ("git", "GitForCausalLM"),
+ ("glm", "GlmForCausalLM"),
+ ("glm4", "Glm4ForCausalLM"),
+ ("glm4_moe", "Glm4MoeForCausalLM"),
+ ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
+ ("glm_moe_dsa", "GlmMoeDsaForCausalLM"),
+ ("got_ocr2", "GotOcr2ForConditionalGeneration"),
+ ("gpt-sw3", "GPT2LMHeadModel"),
+ ("gpt2", "GPT2LMHeadModel"),
+ ("gpt_bigcode", "GPTBigCodeForCausalLM"),
+ ("gpt_neo", "GPTNeoForCausalLM"),
+ ("gpt_neox", "GPTNeoXForCausalLM"),
+ ("gpt_neox_japanese", "GPTNeoXJapaneseForCausalLM"),
+ ("gpt_oss", "GptOssForCausalLM"),
+ ("gptj", "GPTJForCausalLM"),
+ ("granite", "GraniteForCausalLM"),
+ ("granitemoe", "GraniteMoeForCausalLM"),
+ ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
+ ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
+ ("helium", "HeliumForCausalLM"),
+ ("hrm_text", "HrmTextForCausalLM"),
+ ("hunyuan_v1_dense", "HunYuanDenseV1ForCausalLM"),
+ ("hunyuan_v1_moe", "HunYuanMoEV1ForCausalLM"),
+ ("hy_v3", "HYV3ForCausalLM"),
+ ("hyperclovax", "HyperCLOVAXForCausalLM"),
+ ("jais2", "Jais2ForCausalLM"),
+ ("jamba", "JambaForCausalLM"),
+ ("jetmoe", "JetMoeForCausalLM"),
+ ("laguna", "LagunaForCausalLM"),
+ ("lfm2", "Lfm2ForCausalLM"),
+ ("lfm2_moe", "Lfm2MoeForCausalLM"),
+ ("llama", "LlamaForCausalLM"),
+ ("llama4", "Llama4ForCausalLM"),
+ ("llama4_text", "Llama4ForCausalLM"),
+ ("longcat_flash", "LongcatFlashForCausalLM"),
+ ("mamba", "MambaForCausalLM"),
+ ("mamba2", "Mamba2ForCausalLM"),
+ ("marian", "MarianForCausalLM"),
+ ("mbart", "MBartForCausalLM"),
+ ("megatron-bert", "MegatronBertForCausalLM"),
+ ("minimax", "MiniMaxForCausalLM"),
+ ("minimax_m2", "MiniMaxM2ForCausalLM"),
+ ("ministral", "MinistralForCausalLM"),
+ ("ministral3", "Ministral3ForCausalLM"),
+ ("mistral", "MistralForCausalLM"),
+ ("mixtral", "MixtralForCausalLM"),
+ ("mllama", "MllamaForCausalLM"),
+ ("modernbert-decoder", "ModernBertDecoderForCausalLM"),
+ ("moshi", "MoshiForCausalLM"),
+ ("mpt", "MptForCausalLM"),
+ ("musicgen", "MusicgenForCausalLM"),
+ ("musicgen_melody", "MusicgenMelodyForCausalLM"),
+ ("mvp", "MvpForCausalLM"),
+ ("nanochat", "NanoChatForCausalLM"),
+ ("nemotron", "NemotronForCausalLM"),
+ ("nemotron_h", "NemotronHForCausalLM"),
+ ("olmo", "OlmoForCausalLM"),
+ ("olmo2", "Olmo2ForCausalLM"),
+ ("olmo3", "Olmo3ForCausalLM"),
+ ("olmo_hybrid", "OlmoHybridForCausalLM"),
+ ("olmoe", "OlmoeForCausalLM"),
+ ("openai-gpt", "OpenAIGPTLMHeadModel"),
+ ("opt", "OPTForCausalLM"),
+ ("pegasus", "PegasusForCausalLM"),
+ ("persimmon", "PersimmonForCausalLM"),
+ ("phi", "PhiForCausalLM"),
+ ("phi3", "Phi3ForCausalLM"),
+ ("phi4_multimodal", "Phi4MultimodalForCausalLM"),
+ ("phimoe", "PhimoeForCausalLM"),
+ ("plbart", "PLBartForCausalLM"),
+ ("prophetnet", "ProphetNetForCausalLM"),
+ ("qwen2", "Qwen2ForCausalLM"),
+ ("qwen2_moe", "Qwen2MoeForCausalLM"),
+ ("qwen3", "Qwen3ForCausalLM"),
+ ("qwen3_5", "Qwen3_5ForCausalLM"), # VLM compatibility
+ ("qwen3_5_moe", "Qwen3_5MoeForCausalLM"), # VLM compatibility
+ ("qwen3_5_moe_text", "Qwen3_5MoeForCausalLM"),
+ ("qwen3_5_text", "Qwen3_5ForCausalLM"),
+ ("qwen3_moe", "Qwen3MoeForCausalLM"),
+ ("qwen3_next", "Qwen3NextForCausalLM"),
+ ("recurrent_gemma", "RecurrentGemmaForCausalLM"),
+ ("reformer", "ReformerModelWithLMHead"),
+ ("rembert", "RemBertForCausalLM"),
+ ("roberta", "RobertaForCausalLM"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormForCausalLM"),
+ ("roc_bert", "RoCBertForCausalLM"),
+ ("roformer", "RoFormerForCausalLM"),
+ ("rwkv", "RwkvForCausalLM"),
+ ("seed_oss", "SeedOssForCausalLM"),
+ ("smollm3", "SmolLM3ForCausalLM"),
+ ("solar_open", "SolarOpenForCausalLM"),
+ ("stablelm", "StableLmForCausalLM"),
+ ("starcoder2", "Starcoder2ForCausalLM"),
+ ("trocr", "TrOCRForCausalLM"),
+ ("vaultgemma", "VaultGemmaForCausalLM"),
+ ("whisper", "WhisperForCausalLM"),
+ ("xglm", "XGLMForCausalLM"),
+ ("xlm", "XLMWithLMHeadModel"),
+ ("xlm-roberta", "XLMRobertaForCausalLM"),
+ ("xlm-roberta-xl", "XLMRobertaXLForCausalLM"),
+ ("xlnet", "XLNetLMHeadModel"),
+ ("xlstm", "xLSTMForCausalLM"),
+ ("xmod", "XmodForCausalLM"),
+ ("youtu", "YoutuForCausalLM"),
+ ("zamba", "ZambaForCausalLM"),
+ ("zamba2", "Zamba2ForCausalLM"),
+ ]
+)
+
+MODEL_FOR_IMAGE_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Image mapping
+ ("aimv2_vision_model", "Aimv2VisionModel"),
+ ("beit", "BeitModel"),
+ ("bit", "BitModel"),
+ ("cohere2_vision", "Cohere2VisionModel"),
+ ("conditional_detr", "ConditionalDetrModel"),
+ ("convnext", "ConvNextModel"),
+ ("convnextv2", "ConvNextV2Model"),
+ ("dab-detr", "DabDetrModel"),
+ ("data2vec-vision", "Data2VecVisionModel"),
+ ("deformable_detr", "DeformableDetrModel"),
+ ("deit", "DeiTModel"),
+ ("depth_pro", "DepthProModel"),
+ ("detr", "DetrModel"),
+ ("dinat", "DinatModel"),
+ ("dinov2", "Dinov2Model"),
+ ("dinov2_with_registers", "Dinov2WithRegistersModel"),
+ ("dinov3_convnext", "DINOv3ConvNextModel"),
+ ("dinov3_vit", "DINOv3ViTModel"),
+ ("dpt", "DPTModel"),
+ ("efficientnet", "EfficientNetModel"),
+ ("focalnet", "FocalNetModel"),
+ ("glpn", "GLPNModel"),
+ ("hiera", "HieraModel"),
+ ("ijepa", "IJepaModel"),
+ ("imagegpt", "ImageGPTModel"),
+ ("levit", "LevitModel"),
+ ("llama4", "Llama4VisionModel"),
+ ("mlcd", "MLCDVisionModel"), # Keep this to make some original hub repositories (from `DeepGlint-AI`) works
+ ("mlcd_vision_model", "MLCDVisionModel"),
+ ("mllama", "MllamaVisionModel"),
+ ("mobilenet_v1", "MobileNetV1Model"),
+ ("mobilenet_v2", "MobileNetV2Model"),
+ ("mobilevit", "MobileViTModel"),
+ ("mobilevitv2", "MobileViTV2Model"),
+ ("pixio", "PixioModel"),
+ ("poolformer", "PoolFormerModel"),
+ ("pvt", "PvtModel"),
+ ("regnet", "RegNetModel"),
+ ("resnet", "ResNetModel"),
+ ("segformer", "SegformerModel"),
+ ("siglip_vision_model", "SiglipVisionModel"),
+ ("swiftformer", "SwiftFormerModel"),
+ ("swin", "SwinModel"),
+ ("swin2sr", "Swin2SRModel"),
+ ("swinv2", "Swinv2Model"),
+ ("table-transformer", "TableTransformerModel"),
+ ("timesformer", "TimesformerModel"),
+ ("timm_backbone", "TimmBackbone"),
+ ("timm_wrapper", "TimmWrapperModel"),
+ ("videomae", "VideoMAEModel"),
+ ("vit", "ViTModel"),
+ ("vit_mae", "ViTMAEModel"),
+ ("vit_msn", "ViTMSNModel"),
+ ("vitdet", "VitDetModel"),
+ ("vivit", "VivitModel"),
+ ("yolos", "YolosModel"),
+ ]
+)
+
+MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING_NAMES = OrderedDict(
+ [
+ ("deit", "DeiTForMaskedImageModeling"),
+ ("focalnet", "FocalNetForMaskedImageModeling"),
+ ("swin", "SwinForMaskedImageModeling"),
+ ("swinv2", "Swinv2ForMaskedImageModeling"),
+ ("vit", "ViTForMaskedImageModeling"),
+ ]
+)
+
+
+MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING_NAMES = OrderedDict(
+ # Model for Causal Image Modeling mapping
+ [
+ ("imagegpt", "ImageGPTForCausalImageModeling"),
+ ]
+)
+
+MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Image Classification mapping
+ ("beit", "BeitForImageClassification"),
+ ("bit", "BitForImageClassification"),
+ ("clip", "CLIPForImageClassification"),
+ ("convnext", "ConvNextForImageClassification"),
+ ("convnextv2", "ConvNextV2ForImageClassification"),
+ ("cvt", "CvtForImageClassification"),
+ ("data2vec-vision", "Data2VecVisionForImageClassification"),
+ (
+ "deit",
+ ("DeiTForImageClassification", "DeiTForImageClassificationWithTeacher"),
+ ),
+ ("dinat", "DinatForImageClassification"),
+ ("dinov2", "Dinov2ForImageClassification"),
+ ("dinov2_with_registers", "Dinov2WithRegistersForImageClassification"),
+ ("donut-swin", "DonutSwinForImageClassification"),
+ ("efficientnet", "EfficientNetForImageClassification"),
+ ("focalnet", "FocalNetForImageClassification"),
+ ("hgnet_v2", "HGNetV2ForImageClassification"),
+ ("hiera", "HieraForImageClassification"),
+ ("ijepa", "IJepaForImageClassification"),
+ ("imagegpt", "ImageGPTForImageClassification"),
+ (
+ "levit",
+ ("LevitForImageClassification", "LevitForImageClassificationWithTeacher"),
+ ),
+ ("metaclip_2", "MetaClip2ForImageClassification"),
+ ("mobilenet_v1", "MobileNetV1ForImageClassification"),
+ ("mobilenet_v2", "MobileNetV2ForImageClassification"),
+ ("mobilevit", "MobileViTForImageClassification"),
+ ("mobilevitv2", "MobileViTV2ForImageClassification"),
+ (
+ "perceiver",
+ (
+ "PerceiverForImageClassificationLearned",
+ "PerceiverForImageClassificationFourier",
+ "PerceiverForImageClassificationConvProcessing",
+ ),
+ ),
+ ("poolformer", "PoolFormerForImageClassification"),
+ ("pp_lcnet", "PPLCNetForImageClassification"),
+ ("pvt", "PvtForImageClassification"),
+ ("pvt_v2", "PvtV2ForImageClassification"),
+ ("regnet", "RegNetForImageClassification"),
+ ("resnet", "ResNetForImageClassification"),
+ ("segformer", "SegformerForImageClassification"),
+ ("shieldgemma2", "ShieldGemma2ForImageClassification"),
+ ("siglip", "SiglipForImageClassification"),
+ ("siglip2", "Siglip2ForImageClassification"),
+ ("swiftformer", "SwiftFormerForImageClassification"),
+ ("swin", "SwinForImageClassification"),
+ ("swinv2", "Swinv2ForImageClassification"),
+ ("textnet", "TextNetForImageClassification"),
+ ("timm_wrapper", "TimmWrapperForImageClassification"),
+ ("vit", "ViTForImageClassification"),
+ ("vit_msn", "ViTMSNForImageClassification"),
+ ]
+)
+
+MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Do not add new models here, this class will be deprecated in the future.
+ # Model for Image Segmentation mapping
+ ("detr", "DetrForSegmentation"),
+ ]
+)
+
+MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Semantic Segmentation mapping
+ ("beit", "BeitForSemanticSegmentation"),
+ ("data2vec-vision", "Data2VecVisionForSemanticSegmentation"),
+ ("dpt", "DPTForSemanticSegmentation"),
+ ("mobilenet_v2", "MobileNetV2ForSemanticSegmentation"),
+ ("mobilevit", "MobileViTForSemanticSegmentation"),
+ ("mobilevitv2", "MobileViTV2ForSemanticSegmentation"),
+ ("segformer", "SegformerForSemanticSegmentation"),
+ ("upernet", "UperNetForSemanticSegmentation"),
+ ]
+)
+
+MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Instance Segmentation mapping
+ # MaskFormerForInstanceSegmentation can be removed from this mapping in v5
+ ("maskformer", "MaskFormerForInstanceSegmentation"),
+ ("rf_detr", "RfDetrForInstanceSegmentation"),
+ ]
+)
+
+MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Universal Segmentation mapping
+ ("detr", "DetrForSegmentation"),
+ ("eomt", "EomtForUniversalSegmentation"),
+ ("eomt_dinov3", "EomtDinov3ForUniversalSegmentation"),
+ ("mask2former", "Mask2FormerForUniversalSegmentation"),
+ ("maskformer", "MaskFormerForInstanceSegmentation"),
+ ("oneformer", "OneFormerForUniversalSegmentation"),
+ ("videomt", "VideomtForUniversalSegmentation"),
+ ]
+)
+
+MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ ("timesformer", "TimesformerForVideoClassification"),
+ ("videomae", "VideoMAEForVideoClassification"),
+ ("vivit", "VivitForVideoClassification"),
+ ("vjepa2", "VJEPA2ForVideoClassification"),
+ ]
+)
+
+MODEL_FOR_RETRIEVAL_MAPPING_NAMES = OrderedDict(
+ [
+ ("colmodernvbert", "ColModernVBertForRetrieval"),
+ ("colpali", "ColPaliForRetrieval"),
+ ]
+)
+
+MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = OrderedDict(
+ [
+ ("aria", "AriaForConditionalGeneration"),
+ ("aya_vision", "AyaVisionForConditionalGeneration"),
+ ("blip", "BlipForConditionalGeneration"),
+ ("blip-2", "Blip2ForConditionalGeneration"),
+ ("chameleon", "ChameleonForConditionalGeneration"),
+ ("cohere2_vision", "Cohere2VisionForConditionalGeneration"),
+ ("deepseek_vl", "DeepseekVLForConditionalGeneration"),
+ ("deepseek_vl_hybrid", "DeepseekVLHybridForConditionalGeneration"),
+ ("emu3", "Emu3ForConditionalGeneration"),
+ ("ernie4_5_vl_moe", "Ernie4_5_VLMoeForConditionalGeneration"),
+ ("evolla", "EvollaForProteinText2Text"),
+ ("exaone4_5", "Exaone4_5_ForConditionalGeneration"),
+ ("fast_vlm", "FastVlmForConditionalGeneration"),
+ ("florence2", "Florence2ForConditionalGeneration"),
+ ("fuyu", "FuyuForCausalLM"),
+ ("gemma3", "Gemma3ForConditionalGeneration"),
+ ("gemma3n", "Gemma3nForConditionalGeneration"),
+ ("gemma4", "Gemma4ForConditionalGeneration"),
+ ("git", "GitForCausalLM"),
+ ("glm46v", "Glm46VForConditionalGeneration"),
+ ("glm4v", "Glm4vForConditionalGeneration"),
+ ("glm4v_moe", "Glm4vMoeForConditionalGeneration"),
+ ("glm_ocr", "GlmOcrForConditionalGeneration"),
+ ("got_ocr2", "GotOcr2ForConditionalGeneration"),
+ ("granite4_vision", "Granite4VisionForConditionalGeneration"),
+ ("idefics", "IdeficsForVisionText2Text"),
+ ("idefics2", "Idefics2ForConditionalGeneration"),
+ ("idefics3", "Idefics3ForConditionalGeneration"),
+ ("instructblip", "InstructBlipForConditionalGeneration"),
+ ("instructblipvideo", "InstructBlipVideoForConditionalGeneration"),
+ ("internvl", "InternVLForConditionalGeneration"),
+ ("janus", "JanusForConditionalGeneration"),
+ ("kosmos-2", "Kosmos2ForConditionalGeneration"),
+ ("kosmos-2.5", "Kosmos2_5ForConditionalGeneration"),
+ ("lfm2_vl", "Lfm2VlForConditionalGeneration"),
+ ("lighton_ocr", "LightOnOcrForConditionalGeneration"),
+ ("llama4", "Llama4ForConditionalGeneration"),
+ ("llava", "LlavaForConditionalGeneration"),
+ ("llava_next", "LlavaNextForConditionalGeneration"),
+ ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
+ ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
+ ("minicpmv4_6", "MiniCPMV4_6ForConditionalGeneration"),
+ ("mistral3", "Mistral3ForConditionalGeneration"),
+ ("mistral4", "Mistral4ForCausalLM"),
+ ("mllama", "MllamaForConditionalGeneration"),
+ ("ovis2", "Ovis2ForConditionalGeneration"),
+ ("paddleocr_vl", "PaddleOCRVLForConditionalGeneration"),
+ ("paligemma", "PaliGemmaForConditionalGeneration"),
+ ("perception_lm", "PerceptionLMForConditionalGeneration"),
+ ("pi0", "PI0ForConditionalGeneration"),
+ ("pix2struct", "Pix2StructForConditionalGeneration"),
+ ("pp_chart2table", "GotOcr2ForConditionalGeneration"),
+ ("pp_formulanet", "PPFormulaNetForConditionalGeneration"),
+ ("qianfan_ocr", "QianfanOCRForConditionalGeneration"),
+ ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),
+ ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),
+ ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
+ ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
+ ("qwen3_5_moe", "Qwen3_5MoeForConditionalGeneration"),
+ ("qwen3_omni_moe_thinker", "Qwen3OmniMoeThinkerForConditionalGeneration"),
+ ("qwen3_vl", "Qwen3VLForConditionalGeneration"),
+ ("qwen3_vl_moe", "Qwen3VLMoeForConditionalGeneration"),
+ ("shieldgemma2", "Gemma3ForConditionalGeneration"),
+ ("smolvlm", "SmolVLMForConditionalGeneration"),
+ ("t5gemma2", "T5Gemma2ForConditionalGeneration"),
+ ("udop", "UdopForConditionalGeneration"),
+ ("video_llama_3", "VideoLlama3ForConditionalGeneration"),
+ ("video_llava", "VideoLlavaForConditionalGeneration"),
+ ("vipllava", "VipLlavaForConditionalGeneration"),
+ ("vision-encoder-decoder", "VisionEncoderDecoderModel"),
+ ]
+)
+
+# Models that accept text and optionally multimodal data in inputs
+# and can generate text and optionally multimodal data.
+MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES = OrderedDict(
+ [
+ *list(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES.items()),
+ ("glmasr", "GlmAsrForConditionalGeneration"),
+ ("granite_speech", "GraniteSpeechForConditionalGeneration"),
+ ("granite_speech_plus", "GraniteSpeechPlusForConditionalGeneration"),
+ ("kyutai_speech_to_text", "KyutaiSpeechToTextForConditionalGeneration"),
+ ("phi4_multimodal", "Phi4MultimodalForCausalLM"),
+ ("qwen2_5_omni", "Qwen2_5OmniForConditionalGeneration"),
+ ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),
+ ("qwen3_omni_moe", "Qwen3OmniMoeForConditionalGeneration"),
+ ("vibevoice_asr", "VibeVoiceAsrForConditionalGeneration"),
+ ("voxtral", "VoxtralForConditionalGeneration"),
+ ("voxtral_realtime", "VoxtralRealtimeForConditionalGeneration"),
+ ]
+)
+
+
+MODEL_FOR_MASKED_LM_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Masked LM mapping
+ ("albert", "AlbertForMaskedLM"),
+ ("bart", "BartForConditionalGeneration"),
+ ("bert", "BertForMaskedLM"),
+ ("big_bird", "BigBirdForMaskedLM"),
+ ("camembert", "CamembertForMaskedLM"),
+ ("convbert", "ConvBertForMaskedLM"),
+ ("data2vec-text", "Data2VecTextForMaskedLM"),
+ ("deberta", "DebertaForMaskedLM"),
+ ("deberta-v2", "DebertaV2ForMaskedLM"),
+ ("distilbert", "DistilBertForMaskedLM"),
+ ("electra", "ElectraForMaskedLM"),
+ ("ernie", "ErnieForMaskedLM"),
+ ("esm", "EsmForMaskedLM"),
+ ("eurobert", "EuroBertForMaskedLM"),
+ ("flaubert", "FlaubertWithLMHeadModel"),
+ ("fnet", "FNetForMaskedLM"),
+ ("funnel", "FunnelForMaskedLM"),
+ ("ibert", "IBertForMaskedLM"),
+ ("jina_embeddings_v3", "JinaEmbeddingsV3ForMaskedLM"),
+ ("layoutlm", "LayoutLMForMaskedLM"),
+ ("longformer", "LongformerForMaskedLM"),
+ ("luke", "LukeForMaskedLM"),
+ ("mbart", "MBartForConditionalGeneration"),
+ ("megatron-bert", "MegatronBertForMaskedLM"),
+ ("mobilebert", "MobileBertForMaskedLM"),
+ ("modernbert", "ModernBertForMaskedLM"),
+ ("modernvbert", "ModernVBertForMaskedLM"),
+ ("mpnet", "MPNetForMaskedLM"),
+ ("mra", "MraForMaskedLM"),
+ ("mvp", "MvpForConditionalGeneration"),
+ ("nomic_bert", "NomicBertForMaskedLM"),
+ ("nystromformer", "NystromformerForMaskedLM"),
+ ("perceiver", "PerceiverForMaskedLM"),
+ ("reformer", "ReformerForMaskedLM"),
+ ("rembert", "RemBertForMaskedLM"),
+ ("roberta", "RobertaForMaskedLM"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormForMaskedLM"),
+ ("roc_bert", "RoCBertForMaskedLM"),
+ ("roformer", "RoFormerForMaskedLM"),
+ ("squeezebert", "SqueezeBertForMaskedLM"),
+ ("tapas", "TapasForMaskedLM"),
+ ("xlm", "XLMWithLMHeadModel"),
+ ("xlm-roberta", "XLMRobertaForMaskedLM"),
+ ("xlm-roberta-xl", "XLMRobertaXLForMaskedLM"),
+ ("xmod", "XmodForMaskedLM"),
+ ("yoso", "YosoForMaskedLM"),
+ ]
+)
+
+MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Object Detection mapping
+ ("conditional_detr", "ConditionalDetrForObjectDetection"),
+ ("d_fine", "DFineForObjectDetection"),
+ ("dab-detr", "DabDetrForObjectDetection"),
+ ("deformable_detr", "DeformableDetrForObjectDetection"),
+ ("deimv2", "Deimv2ForObjectDetection"),
+ ("detr", "DetrForObjectDetection"),
+ ("lw_detr", "LwDetrForObjectDetection"),
+ ("pp_doclayout_v2", "PPDocLayoutV2ForObjectDetection"),
+ ("pp_doclayout_v3", "PPDocLayoutV3ForObjectDetection"),
+ ("pp_ocrv5_mobile_det", "PPOCRV5MobileDetForObjectDetection"),
+ ("pp_ocrv5_server_det", "PPOCRV5ServerDetForObjectDetection"),
+ ("rf_detr", "RfDetrForObjectDetection"),
+ ("rt_detr", "RTDetrForObjectDetection"),
+ ("rt_detr_v2", "RTDetrV2ForObjectDetection"),
+ ("table-transformer", "TableTransformerForObjectDetection"),
+ ("yolos", "YolosForObjectDetection"),
+ ]
+)
+
+MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Zero Shot Object Detection mapping
+ ("grounding-dino", "GroundingDinoForObjectDetection"),
+ ("mm-grounding-dino", "MMGroundingDinoForObjectDetection"),
+ ("omdet-turbo", "OmDetTurboForObjectDetection"),
+ ("owlv2", "Owlv2ForObjectDetection"),
+ ("owlvit", "OwlViTForObjectDetection"),
+ ]
+)
+
+MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for depth estimation mapping
+ ("chmv2", "CHMv2ForDepthEstimation"),
+ ("depth_anything", "DepthAnythingForDepthEstimation"),
+ ("depth_pro", "DepthProForDepthEstimation"),
+ ("dpt", "DPTForDepthEstimation"),
+ ("glpn", "GLPNForDepthEstimation"),
+ ("prompt_depth_anything", "PromptDepthAnythingForDepthEstimation"),
+ ("zoedepth", "ZoeDepthForDepthEstimation"),
+ ]
+)
+
+
+MODEL_FOR_TEXT_RECOGNITION_MAPPING_NAMES = OrderedDict(
+ [
+ ("pp_ocrv5_mobile_rec", "PPOCRV5MobileRecForTextRecognition"),
+ ("pp_ocrv5_server_rec", "PPOCRV5ServerRecForTextRecognition"),
+ ]
+)
+
+
+MODEL_FOR_TABLE_RECOGNITION_MAPPING_NAMES = OrderedDict(
+ [
+ ("slanet", "SLANetForTableRecognition"),
+ ("slanext", "SLANeXtForTableRecognition"),
+ ]
+)
+
+
+MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Seq2Seq Causal LM mapping
+ ("audioflamingo3", "AudioFlamingo3ForConditionalGeneration"),
+ ("bart", "BartForConditionalGeneration"),
+ ("bigbird_pegasus", "BigBirdPegasusForConditionalGeneration"),
+ ("blenderbot", "BlenderbotForConditionalGeneration"),
+ ("blenderbot-small", "BlenderbotSmallForConditionalGeneration"),
+ ("encoder-decoder", "EncoderDecoderModel"),
+ ("fsmt", "FSMTForConditionalGeneration"),
+ ("glmasr", "GlmAsrForConditionalGeneration"),
+ ("granite_speech", "GraniteSpeechForConditionalGeneration"),
+ ("granite_speech_plus", "GraniteSpeechPlusForConditionalGeneration"),
+ ("led", "LEDForConditionalGeneration"),
+ ("longt5", "LongT5ForConditionalGeneration"),
+ ("m2m_100", "M2M100ForConditionalGeneration"),
+ ("marian", "MarianMTModel"),
+ ("mbart", "MBartForConditionalGeneration"),
+ ("mt5", "MT5ForConditionalGeneration"),
+ ("musicflamingo", "MusicFlamingoForConditionalGeneration"),
+ ("mvp", "MvpForConditionalGeneration"),
+ ("nllb-moe", "NllbMoeForConditionalGeneration"),
+ ("pegasus", "PegasusForConditionalGeneration"),
+ ("pegasus_x", "PegasusXForConditionalGeneration"),
+ ("plbart", "PLBartForConditionalGeneration"),
+ ("prophetnet", "ProphetNetForConditionalGeneration"),
+ ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),
+ ("seamless_m4t", "SeamlessM4TForTextToText"),
+ ("seamless_m4t_v2", "SeamlessM4Tv2ForTextToText"),
+ ("switch_transformers", "SwitchTransformersForConditionalGeneration"),
+ ("t5", "T5ForConditionalGeneration"),
+ ("t5gemma", "T5GemmaForConditionalGeneration"),
+ ("t5gemma2", "T5Gemma2ForConditionalGeneration"),
+ ("umt5", "UMT5ForConditionalGeneration"),
+ ("vibevoice_asr", "VibeVoiceAsrForConditionalGeneration"),
+ ("voxtral", "VoxtralForConditionalGeneration"),
+ ("voxtral_realtime", "VoxtralRealtimeForConditionalGeneration"),
+ ]
+)
+
+
+MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES = OrderedDict(
+ [
+ ("cohere_asr", "CohereAsrForConditionalGeneration"),
+ ("dia", "DiaForConditionalGeneration"),
+ ("granite_speech", "GraniteSpeechForConditionalGeneration"),
+ ("granite_speech_plus", "GraniteSpeechPlusForConditionalGeneration"),
+ ("kyutai_speech_to_text", "KyutaiSpeechToTextForConditionalGeneration"),
+ ("moonshine", "MoonshineForConditionalGeneration"),
+ ("moonshine_streaming", "MoonshineStreamingForConditionalGeneration"),
+ ("pop2piano", "Pop2PianoForConditionalGeneration"),
+ ("seamless_m4t", "SeamlessM4TForSpeechToText"),
+ ("seamless_m4t_v2", "SeamlessM4Tv2ForSpeechToText"),
+ ("speech-encoder-decoder", "SpeechEncoderDecoderModel"),
+ ("speech_to_text", "Speech2TextForConditionalGeneration"),
+ ("speecht5", "SpeechT5ForSpeechToText"),
+ ("vibevoice_asr", "VibeVoiceAsrForConditionalGeneration"),
+ ("voxtral", "VoxtralForConditionalGeneration"),
+ ("voxtral_realtime", "VoxtralRealtimeForConditionalGeneration"),
+ ("whisper", "WhisperForConditionalGeneration"),
+ ]
+)
+
+MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Sequence Classification mapping
+ ("albert", "AlbertForSequenceClassification"),
+ ("arcee", "ArceeForSequenceClassification"),
+ ("bart", "BartForSequenceClassification"),
+ ("bert", "BertForSequenceClassification"),
+ ("big_bird", "BigBirdForSequenceClassification"),
+ ("bigbird_pegasus", "BigBirdPegasusForSequenceClassification"),
+ ("biogpt", "BioGptForSequenceClassification"),
+ ("bloom", "BloomForSequenceClassification"),
+ ("camembert", "CamembertForSequenceClassification"),
+ ("canine", "CanineForSequenceClassification"),
+ ("convbert", "ConvBertForSequenceClassification"),
+ ("ctrl", "CTRLForSequenceClassification"),
+ ("data2vec-text", "Data2VecTextForSequenceClassification"),
+ ("deberta", "DebertaForSequenceClassification"),
+ ("deberta-v2", "DebertaV2ForSequenceClassification"),
+ ("deepseek_v2", "DeepseekV2ForSequenceClassification"),
+ ("deepseek_v3", "DeepseekV3ForSequenceClassification"),
+ ("diffllama", "DiffLlamaForSequenceClassification"),
+ ("distilbert", "DistilBertForSequenceClassification"),
+ ("doge", "DogeForSequenceClassification"),
+ ("electra", "ElectraForSequenceClassification"),
+ ("ernie", "ErnieForSequenceClassification"),
+ ("esm", "EsmForSequenceClassification"),
+ ("eurobert", "EuroBertForSequenceClassification"),
+ ("exaone4", "Exaone4ForSequenceClassification"),
+ ("falcon", "FalconForSequenceClassification"),
+ ("flaubert", "FlaubertForSequenceClassification"),
+ ("fnet", "FNetForSequenceClassification"),
+ ("funnel", "FunnelForSequenceClassification"),
+ ("gemma", "GemmaForSequenceClassification"),
+ ("gemma2", "Gemma2ForSequenceClassification"),
+ ("gemma3", "Gemma3ForSequenceClassification"),
+ ("gemma3_text", "Gemma3TextForSequenceClassification"),
+ ("glm", "GlmForSequenceClassification"),
+ ("glm4", "Glm4ForSequenceClassification"),
+ ("gpt-sw3", "GPT2ForSequenceClassification"),
+ ("gpt2", "GPT2ForSequenceClassification"),
+ ("gpt_bigcode", "GPTBigCodeForSequenceClassification"),
+ ("gpt_neo", "GPTNeoForSequenceClassification"),
+ ("gpt_neox", "GPTNeoXForSequenceClassification"),
+ ("gpt_oss", "GptOssForSequenceClassification"),
+ ("gptj", "GPTJForSequenceClassification"),
+ ("helium", "HeliumForSequenceClassification"),
+ ("hunyuan_v1_dense", "HunYuanDenseV1ForSequenceClassification"),
+ ("hunyuan_v1_moe", "HunYuanMoEV1ForSequenceClassification"),
+ ("ibert", "IBertForSequenceClassification"),
+ ("jamba", "JambaForSequenceClassification"),
+ ("jetmoe", "JetMoeForSequenceClassification"),
+ ("jina_embeddings_v3", "JinaEmbeddingsV3ForSequenceClassification"),
+ ("layoutlm", "LayoutLMForSequenceClassification"),
+ ("layoutlmv2", "LayoutLMv2ForSequenceClassification"),
+ ("layoutlmv3", "LayoutLMv3ForSequenceClassification"),
+ ("lilt", "LiltForSequenceClassification"),
+ ("llama", "LlamaForSequenceClassification"),
+ ("longformer", "LongformerForSequenceClassification"),
+ ("luke", "LukeForSequenceClassification"),
+ ("markuplm", "MarkupLMForSequenceClassification"),
+ ("mbart", "MBartForSequenceClassification"),
+ ("megatron-bert", "MegatronBertForSequenceClassification"),
+ ("minimax", "MiniMaxForSequenceClassification"),
+ ("ministral", "MinistralForSequenceClassification"),
+ ("ministral3", "Ministral3ForSequenceClassification"),
+ ("mistral", "MistralForSequenceClassification"),
+ ("mistral4", "Mistral4ForSequenceClassification"),
+ ("mixtral", "MixtralForSequenceClassification"),
+ ("mobilebert", "MobileBertForSequenceClassification"),
+ ("modernbert", "ModernBertForSequenceClassification"),
+ ("modernbert-decoder", "ModernBertDecoderForSequenceClassification"),
+ ("modernvbert", "ModernVBertForSequenceClassification"),
+ ("mpnet", "MPNetForSequenceClassification"),
+ ("mpt", "MptForSequenceClassification"),
+ ("mra", "MraForSequenceClassification"),
+ ("mt5", "MT5ForSequenceClassification"),
+ ("mvp", "MvpForSequenceClassification"),
+ ("nemotron", "NemotronForSequenceClassification"),
+ ("nomic_bert", "NomicBertForSequenceClassification"),
+ ("nystromformer", "NystromformerForSequenceClassification"),
+ ("olmo", "OlmoForSequenceClassification"),
+ ("olmo2", "Olmo2ForSequenceClassification"),
+ ("olmo3", "Olmo3ForSequenceClassification"),
+ ("openai-gpt", "OpenAIGPTForSequenceClassification"),
+ ("opt", "OPTForSequenceClassification"),
+ ("perceiver", "PerceiverForSequenceClassification"),
+ ("persimmon", "PersimmonForSequenceClassification"),
+ ("phi", "PhiForSequenceClassification"),
+ ("phi3", "Phi3ForSequenceClassification"),
+ ("phimoe", "PhimoeForSequenceClassification"),
+ ("plbart", "PLBartForSequenceClassification"),
+ ("qwen2", "Qwen2ForSequenceClassification"),
+ ("qwen2_moe", "Qwen2MoeForSequenceClassification"),
+ ("qwen3", "Qwen3ForSequenceClassification"),
+ ("qwen3_5", "Qwen3_5ForSequenceClassification"),
+ ("qwen3_5_text", "Qwen3_5TextForSequenceClassification"),
+ ("qwen3_moe", "Qwen3MoeForSequenceClassification"),
+ ("qwen3_next", "Qwen3NextForSequenceClassification"),
+ ("reformer", "ReformerForSequenceClassification"),
+ ("rembert", "RemBertForSequenceClassification"),
+ ("roberta", "RobertaForSequenceClassification"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormForSequenceClassification"),
+ ("roc_bert", "RoCBertForSequenceClassification"),
+ ("roformer", "RoFormerForSequenceClassification"),
+ ("seed_oss", "SeedOssForSequenceClassification"),
+ ("smollm3", "SmolLM3ForSequenceClassification"),
+ ("squeezebert", "SqueezeBertForSequenceClassification"),
+ ("stablelm", "StableLmForSequenceClassification"),
+ ("starcoder2", "Starcoder2ForSequenceClassification"),
+ ("t5", "T5ForSequenceClassification"),
+ ("t5gemma", "T5GemmaForSequenceClassification"),
+ ("t5gemma2", "T5Gemma2ForSequenceClassification"),
+ ("tapas", "TapasForSequenceClassification"),
+ ("umt5", "UMT5ForSequenceClassification"),
+ ("xlm", "XLMForSequenceClassification"),
+ ("xlm-roberta", "XLMRobertaForSequenceClassification"),
+ ("xlm-roberta-xl", "XLMRobertaXLForSequenceClassification"),
+ ("xlnet", "XLNetForSequenceClassification"),
+ ("xmod", "XmodForSequenceClassification"),
+ ("yoso", "YosoForSequenceClassification"),
+ ("zamba", "ZambaForSequenceClassification"),
+ ("zamba2", "Zamba2ForSequenceClassification"),
+ ]
+)
+
+MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Question Answering mapping
+ ("albert", "AlbertForQuestionAnswering"),
+ ("arcee", "ArceeForQuestionAnswering"),
+ ("bart", "BartForQuestionAnswering"),
+ ("bert", "BertForQuestionAnswering"),
+ ("big_bird", "BigBirdForQuestionAnswering"),
+ ("bigbird_pegasus", "BigBirdPegasusForQuestionAnswering"),
+ ("bloom", "BloomForQuestionAnswering"),
+ ("camembert", "CamembertForQuestionAnswering"),
+ ("canine", "CanineForQuestionAnswering"),
+ ("convbert", "ConvBertForQuestionAnswering"),
+ ("data2vec-text", "Data2VecTextForQuestionAnswering"),
+ ("deberta", "DebertaForQuestionAnswering"),
+ ("deberta-v2", "DebertaV2ForQuestionAnswering"),
+ ("diffllama", "DiffLlamaForQuestionAnswering"),
+ ("distilbert", "DistilBertForQuestionAnswering"),
+ ("electra", "ElectraForQuestionAnswering"),
+ ("ernie", "ErnieForQuestionAnswering"),
+ ("exaone4", "Exaone4ForQuestionAnswering"),
+ ("falcon", "FalconForQuestionAnswering"),
+ ("flaubert", "FlaubertForQuestionAnsweringSimple"),
+ ("fnet", "FNetForQuestionAnswering"),
+ ("funnel", "FunnelForQuestionAnswering"),
+ ("gpt2", "GPT2ForQuestionAnswering"),
+ ("gpt_neo", "GPTNeoForQuestionAnswering"),
+ ("gpt_neox", "GPTNeoXForQuestionAnswering"),
+ ("gptj", "GPTJForQuestionAnswering"),
+ ("ibert", "IBertForQuestionAnswering"),
+ ("jina_embeddings_v3", "JinaEmbeddingsV3ForQuestionAnswering"),
+ ("layoutlmv2", "LayoutLMv2ForQuestionAnswering"),
+ ("layoutlmv3", "LayoutLMv3ForQuestionAnswering"),
+ ("led", "LEDForQuestionAnswering"),
+ ("lilt", "LiltForQuestionAnswering"),
+ ("llama", "LlamaForQuestionAnswering"),
+ ("longformer", "LongformerForQuestionAnswering"),
+ ("luke", "LukeForQuestionAnswering"),
+ ("lxmert", "LxmertForQuestionAnswering"),
+ ("markuplm", "MarkupLMForQuestionAnswering"),
+ ("mbart", "MBartForQuestionAnswering"),
+ ("megatron-bert", "MegatronBertForQuestionAnswering"),
+ ("minimax", "MiniMaxForQuestionAnswering"),
+ ("ministral", "MinistralForQuestionAnswering"),
+ ("ministral3", "Ministral3ForQuestionAnswering"),
+ ("mistral", "MistralForQuestionAnswering"),
+ ("mixtral", "MixtralForQuestionAnswering"),
+ ("mobilebert", "MobileBertForQuestionAnswering"),
+ ("modernbert", "ModernBertForQuestionAnswering"),
+ ("mpnet", "MPNetForQuestionAnswering"),
+ ("mpt", "MptForQuestionAnswering"),
+ ("mra", "MraForQuestionAnswering"),
+ ("mt5", "MT5ForQuestionAnswering"),
+ ("mvp", "MvpForQuestionAnswering"),
+ ("nemotron", "NemotronForQuestionAnswering"),
+ ("nystromformer", "NystromformerForQuestionAnswering"),
+ ("opt", "OPTForQuestionAnswering"),
+ ("qwen2", "Qwen2ForQuestionAnswering"),
+ ("qwen2_moe", "Qwen2MoeForQuestionAnswering"),
+ ("qwen3", "Qwen3ForQuestionAnswering"),
+ ("qwen3_moe", "Qwen3MoeForQuestionAnswering"),
+ ("qwen3_next", "Qwen3NextForQuestionAnswering"),
+ ("reformer", "ReformerForQuestionAnswering"),
+ ("rembert", "RemBertForQuestionAnswering"),
+ ("roberta", "RobertaForQuestionAnswering"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormForQuestionAnswering"),
+ ("roc_bert", "RoCBertForQuestionAnswering"),
+ ("roformer", "RoFormerForQuestionAnswering"),
+ ("seed_oss", "SeedOssForQuestionAnswering"),
+ ("smollm3", "SmolLM3ForQuestionAnswering"),
+ ("splinter", "SplinterForQuestionAnswering"),
+ ("squeezebert", "SqueezeBertForQuestionAnswering"),
+ ("t5", "T5ForQuestionAnswering"),
+ ("umt5", "UMT5ForQuestionAnswering"),
+ ("xlm", "XLMForQuestionAnsweringSimple"),
+ ("xlm-roberta", "XLMRobertaForQuestionAnswering"),
+ ("xlm-roberta-xl", "XLMRobertaXLForQuestionAnswering"),
+ ("xlnet", "XLNetForQuestionAnsweringSimple"),
+ ("xmod", "XmodForQuestionAnswering"),
+ ("yoso", "YosoForQuestionAnswering"),
+ ]
+)
+
+MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Table Question Answering mapping
+ ("tapas", "TapasForQuestionAnswering"),
+ ]
+)
+
+MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES = OrderedDict(
+ [
+ ("blip", "BlipForQuestionAnswering"),
+ ("blip-2", "Blip2ForConditionalGeneration"),
+ ("vilt", "ViltForQuestionAnswering"),
+ ]
+)
+
+MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES = OrderedDict(
+ [
+ ("layoutlm", "LayoutLMForQuestionAnswering"),
+ ("layoutlmv2", "LayoutLMv2ForQuestionAnswering"),
+ ("layoutlmv3", "LayoutLMv3ForQuestionAnswering"),
+ ]
+)
+
+MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Token Classification mapping
+ ("albert", "AlbertForTokenClassification"),
+ ("apertus", "ApertusForTokenClassification"),
+ ("arcee", "ArceeForTokenClassification"),
+ ("bert", "BertForTokenClassification"),
+ ("big_bird", "BigBirdForTokenClassification"),
+ ("biogpt", "BioGptForTokenClassification"),
+ ("bloom", "BloomForTokenClassification"),
+ ("bros", "BrosForTokenClassification"),
+ ("camembert", "CamembertForTokenClassification"),
+ ("canine", "CanineForTokenClassification"),
+ ("convbert", "ConvBertForTokenClassification"),
+ ("data2vec-text", "Data2VecTextForTokenClassification"),
+ ("deberta", "DebertaForTokenClassification"),
+ ("deberta-v2", "DebertaV2ForTokenClassification"),
+ ("deepseek_v3", "DeepseekV3ForTokenClassification"),
+ ("diffllama", "DiffLlamaForTokenClassification"),
+ ("distilbert", "DistilBertForTokenClassification"),
+ ("electra", "ElectraForTokenClassification"),
+ ("ernie", "ErnieForTokenClassification"),
+ ("esm", "EsmForTokenClassification"),
+ ("eurobert", "EuroBertForTokenClassification"),
+ ("exaone4", "Exaone4ForTokenClassification"),
+ ("falcon", "FalconForTokenClassification"),
+ ("flaubert", "FlaubertForTokenClassification"),
+ ("fnet", "FNetForTokenClassification"),
+ ("funnel", "FunnelForTokenClassification"),
+ ("gemma", "GemmaForTokenClassification"),
+ ("gemma2", "Gemma2ForTokenClassification"),
+ ("glm", "GlmForTokenClassification"),
+ ("glm4", "Glm4ForTokenClassification"),
+ ("gpt-sw3", "GPT2ForTokenClassification"),
+ ("gpt2", "GPT2ForTokenClassification"),
+ ("gpt_bigcode", "GPTBigCodeForTokenClassification"),
+ ("gpt_neo", "GPTNeoForTokenClassification"),
+ ("gpt_neox", "GPTNeoXForTokenClassification"),
+ ("gpt_oss", "GptOssForTokenClassification"),
+ ("helium", "HeliumForTokenClassification"),
+ ("ibert", "IBertForTokenClassification"),
+ ("jina_embeddings_v3", "JinaEmbeddingsV3ForTokenClassification"),
+ ("layoutlm", "LayoutLMForTokenClassification"),
+ ("layoutlmv2", "LayoutLMv2ForTokenClassification"),
+ ("layoutlmv3", "LayoutLMv3ForTokenClassification"),
+ ("lilt", "LiltForTokenClassification"),
+ ("llama", "LlamaForTokenClassification"),
+ ("longformer", "LongformerForTokenClassification"),
+ ("luke", "LukeForTokenClassification"),
+ ("markuplm", "MarkupLMForTokenClassification"),
+ ("megatron-bert", "MegatronBertForTokenClassification"),
+ ("minimax", "MiniMaxForTokenClassification"),
+ ("ministral", "MinistralForTokenClassification"),
+ ("ministral3", "Ministral3ForTokenClassification"),
+ ("mistral", "MistralForTokenClassification"),
+ ("mistral4", "Mistral4ForTokenClassification"),
+ ("mixtral", "MixtralForTokenClassification"),
+ ("mobilebert", "MobileBertForTokenClassification"),
+ ("modernbert", "ModernBertForTokenClassification"),
+ ("modernvbert", "ModernVBertForTokenClassification"),
+ ("mpnet", "MPNetForTokenClassification"),
+ ("mpt", "MptForTokenClassification"),
+ ("mra", "MraForTokenClassification"),
+ ("mt5", "MT5ForTokenClassification"),
+ ("nemotron", "NemotronForTokenClassification"),
+ ("nomic_bert", "NomicBertForTokenClassification"),
+ ("nystromformer", "NystromformerForTokenClassification"),
+ ("openai_privacy_filter", "OpenAIPrivacyFilterForTokenClassification"),
+ ("persimmon", "PersimmonForTokenClassification"),
+ ("phi", "PhiForTokenClassification"),
+ ("phi3", "Phi3ForTokenClassification"),
+ ("qwen2", "Qwen2ForTokenClassification"),
+ ("qwen2_moe", "Qwen2MoeForTokenClassification"),
+ ("qwen3", "Qwen3ForTokenClassification"),
+ ("qwen3_5", "Qwen3_5ForTokenClassification"),
+ ("qwen3_moe", "Qwen3MoeForTokenClassification"),
+ ("qwen3_next", "Qwen3NextForTokenClassification"),
+ ("rembert", "RemBertForTokenClassification"),
+ ("roberta", "RobertaForTokenClassification"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormForTokenClassification"),
+ ("roc_bert", "RoCBertForTokenClassification"),
+ ("roformer", "RoFormerForTokenClassification"),
+ ("seed_oss", "SeedOssForTokenClassification"),
+ ("smollm3", "SmolLM3ForTokenClassification"),
+ ("squeezebert", "SqueezeBertForTokenClassification"),
+ ("stablelm", "StableLmForTokenClassification"),
+ ("starcoder2", "Starcoder2ForTokenClassification"),
+ ("t5", "T5ForTokenClassification"),
+ ("t5gemma", "T5GemmaForTokenClassification"),
+ ("t5gemma2", "T5Gemma2ForTokenClassification"),
+ ("umt5", "UMT5ForTokenClassification"),
+ ("xlm", "XLMForTokenClassification"),
+ ("xlm-roberta", "XLMRobertaForTokenClassification"),
+ ("xlm-roberta-xl", "XLMRobertaXLForTokenClassification"),
+ ("xlnet", "XLNetForTokenClassification"),
+ ("xmod", "XmodForTokenClassification"),
+ ("yoso", "YosoForTokenClassification"),
+ ]
+)
+
+MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Multiple Choice mapping
+ ("albert", "AlbertForMultipleChoice"),
+ ("bert", "BertForMultipleChoice"),
+ ("big_bird", "BigBirdForMultipleChoice"),
+ ("camembert", "CamembertForMultipleChoice"),
+ ("canine", "CanineForMultipleChoice"),
+ ("convbert", "ConvBertForMultipleChoice"),
+ ("data2vec-text", "Data2VecTextForMultipleChoice"),
+ ("deberta-v2", "DebertaV2ForMultipleChoice"),
+ ("distilbert", "DistilBertForMultipleChoice"),
+ ("electra", "ElectraForMultipleChoice"),
+ ("ernie", "ErnieForMultipleChoice"),
+ ("flaubert", "FlaubertForMultipleChoice"),
+ ("fnet", "FNetForMultipleChoice"),
+ ("funnel", "FunnelForMultipleChoice"),
+ ("ibert", "IBertForMultipleChoice"),
+ ("longformer", "LongformerForMultipleChoice"),
+ ("luke", "LukeForMultipleChoice"),
+ ("megatron-bert", "MegatronBertForMultipleChoice"),
+ ("mobilebert", "MobileBertForMultipleChoice"),
+ ("modernbert", "ModernBertForMultipleChoice"),
+ ("mpnet", "MPNetForMultipleChoice"),
+ ("mra", "MraForMultipleChoice"),
+ ("nystromformer", "NystromformerForMultipleChoice"),
+ ("rembert", "RemBertForMultipleChoice"),
+ ("roberta", "RobertaForMultipleChoice"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormForMultipleChoice"),
+ ("roc_bert", "RoCBertForMultipleChoice"),
+ ("roformer", "RoFormerForMultipleChoice"),
+ ("squeezebert", "SqueezeBertForMultipleChoice"),
+ ("xlm", "XLMForMultipleChoice"),
+ ("xlm-roberta", "XLMRobertaForMultipleChoice"),
+ ("xlm-roberta-xl", "XLMRobertaXLForMultipleChoice"),
+ ("xlnet", "XLNetForMultipleChoice"),
+ ("xmod", "XmodForMultipleChoice"),
+ ("yoso", "YosoForMultipleChoice"),
+ ]
+)
+
+MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES = OrderedDict(
+ [
+ ("bert", "BertForNextSentencePrediction"),
+ ("ernie", "ErnieForNextSentencePrediction"),
+ ("fnet", "FNetForNextSentencePrediction"),
+ ("megatron-bert", "MegatronBertForNextSentencePrediction"),
+ ("mobilebert", "MobileBertForNextSentencePrediction"),
+ ]
+)
+
+MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Audio Classification mapping
+ ("audio-spectrogram-transformer", "ASTForAudioClassification"),
+ ("data2vec-audio", "Data2VecAudioForSequenceClassification"),
+ ("hubert", "HubertForSequenceClassification"),
+ ("sew", "SEWForSequenceClassification"),
+ ("sew-d", "SEWDForSequenceClassification"),
+ ("unispeech", "UniSpeechForSequenceClassification"),
+ ("unispeech-sat", "UniSpeechSatForSequenceClassification"),
+ ("wav2vec2", "Wav2Vec2ForSequenceClassification"),
+ ("wav2vec2-bert", "Wav2Vec2BertForSequenceClassification"),
+ ("wav2vec2-conformer", "Wav2Vec2ConformerForSequenceClassification"),
+ ("wavlm", "WavLMForSequenceClassification"),
+ ("whisper", "WhisperForAudioClassification"),
+ ]
+)
+
+MODEL_FOR_CTC_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Connectionist temporal classification (CTC) mapping
+ ("data2vec-audio", "Data2VecAudioForCTC"),
+ ("hubert", "HubertForCTC"),
+ ("lasr_ctc", "LasrForCTC"),
+ ("parakeet_ctc", "ParakeetForCTC"),
+ ("sew", "SEWForCTC"),
+ ("sew-d", "SEWDForCTC"),
+ ("unispeech", "UniSpeechForCTC"),
+ ("unispeech-sat", "UniSpeechSatForCTC"),
+ ("wav2vec2", "Wav2Vec2ForCTC"),
+ ("wav2vec2-bert", "Wav2Vec2BertForCTC"),
+ ("wav2vec2-conformer", "Wav2Vec2ConformerForCTC"),
+ ("wavlm", "WavLMForCTC"),
+ ]
+)
+
+MODEL_FOR_TDT_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Token-and-Duration Transducer (TDT) mapping.
+ ("parakeet_tdt", "ParakeetForTDT"),
+ ]
+)
+
+
+MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Audio Classification mapping
+ ("data2vec-audio", "Data2VecAudioForAudioFrameClassification"),
+ ("unispeech-sat", "UniSpeechSatForAudioFrameClassification"),
+ ("wav2vec2", "Wav2Vec2ForAudioFrameClassification"),
+ ("wav2vec2-bert", "Wav2Vec2BertForAudioFrameClassification"),
+ ("wav2vec2-conformer", "Wav2Vec2ConformerForAudioFrameClassification"),
+ ("wavlm", "WavLMForAudioFrameClassification"),
+ ]
+)
+
+MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Audio Classification mapping
+ ("data2vec-audio", "Data2VecAudioForXVector"),
+ ("unispeech-sat", "UniSpeechSatForXVector"),
+ ("wav2vec2", "Wav2Vec2ForXVector"),
+ ("wav2vec2-bert", "Wav2Vec2BertForXVector"),
+ ("wav2vec2-conformer", "Wav2Vec2ConformerForXVector"),
+ ("wavlm", "WavLMForXVector"),
+ ]
+)
+
+MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Text-To-Spectrogram mapping
+ ("fastspeech2_conformer", "FastSpeech2ConformerModel"),
+ ("speecht5", "SpeechT5ForTextToSpeech"),
+ ]
+)
+
+MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Text-To-Waveform mapping
+ ("bark", "BarkModel"),
+ ("csm", "CsmForConditionalGeneration"),
+ ("fastspeech2_conformer_with_hifigan", "FastSpeech2ConformerWithHifiGan"),
+ ("higgs_audio_v2", "HiggsAudioV2ForConditionalGeneration"),
+ ("musicgen", "MusicgenForConditionalGeneration"),
+ ("musicgen_melody", "MusicgenMelodyForConditionalGeneration"),
+ ("qwen2_5_omni", "Qwen2_5OmniForConditionalGeneration"),
+ ("qwen3_omni_moe", "Qwen3OmniMoeForConditionalGeneration"),
+ ("seamless_m4t", "SeamlessM4TForTextToSpeech"),
+ ("seamless_m4t_v2", "SeamlessM4Tv2ForTextToSpeech"),
+ ("vits", "VitsModel"),
+ ]
+)
+
+MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ # Model for Zero Shot Image Classification mapping
+ ("align", "AlignModel"),
+ ("altclip", "AltCLIPModel"),
+ ("blip", "BlipModel"),
+ ("blip-2", "Blip2ForImageTextRetrieval"),
+ ("chinese_clip", "ChineseCLIPModel"),
+ ("clip", "CLIPModel"),
+ ("clipseg", "CLIPSegModel"),
+ ("metaclip_2", "MetaClip2Model"),
+ ("siglip", "SiglipModel"),
+ ("siglip2", "Siglip2Model"),
+ ]
+)
+
+MODEL_FOR_BACKBONE_MAPPING_NAMES = OrderedDict(
+ [
+ # Backbone mapping
+ ("beit", "BeitBackbone"),
+ ("bit", "BitBackbone"),
+ ("convnext", "ConvNextBackbone"),
+ ("convnextv2", "ConvNextV2Backbone"),
+ ("dinat", "DinatBackbone"),
+ ("dinov2", "Dinov2Backbone"),
+ ("dinov2_with_registers", "Dinov2WithRegistersBackbone"),
+ ("dinov3_convnext", "DINOv3ConvNextBackbone"),
+ ("dinov3_vit", "DINOv3ViTBackbone"),
+ ("focalnet", "FocalNetBackbone"),
+ ("hgnet_v2", "HGNetV2Backbone"),
+ ("hiera", "HieraBackbone"),
+ ("lw_detr_vit", "LwDetrViTBackbone"),
+ ("maskformer-swin", "MaskFormerSwinBackbone"),
+ ("pixio", "PixioBackbone"),
+ ("pp_lcnet", "PPLCNetBackbone"),
+ ("pp_lcnet_v3", "PPLCNetV3Backbone"),
+ ("pvt_v2", "PvtV2Backbone"),
+ ("resnet", "ResNetBackbone"),
+ ("rf_detr_dinov2", "RfDetrDinov2Backbone"),
+ ("rt_detr_resnet", "RTDetrResNetBackbone"),
+ ("swin", "SwinBackbone"),
+ ("swinv2", "Swinv2Backbone"),
+ ("textnet", "TextNetBackbone"),
+ ("timm_backbone", "TimmBackbone"),
+ ("uvdoc_backbone", "UVDocBackbone"),
+ ("vitdet", "VitDetBackbone"),
+ ("vitpose_backbone", "VitPoseBackbone"),
+ ]
+)
+
+MODEL_FOR_MASK_GENERATION_MAPPING_NAMES = OrderedDict(
+ [
+ ("edgetam", "EdgeTamModel"),
+ ("edgetam_video", "EdgeTamModel"),
+ ("sam", "SamModel"),
+ ("sam2", "Sam2Model"),
+ ("sam2_video", "Sam2Model"),
+ ("sam3_tracker", "Sam3TrackerModel"),
+ ("sam3_video", "Sam3TrackerModel"),
+ ("sam_hq", "SamHQModel"),
+ ]
+)
+
+
+MODEL_FOR_KEYPOINT_DETECTION_MAPPING_NAMES = OrderedDict(
+ [
+ ("superpoint", "SuperPointForKeypointDetection"),
+ ]
+)
+
+MODEL_FOR_KEYPOINT_MATCHING_MAPPING_NAMES = OrderedDict(
+ [
+ ("efficientloftr", "EfficientLoFTRForKeypointMatching"),
+ ("lightglue", "LightGlueForKeypointMatching"),
+ ("superglue", "SuperGlueForKeypointMatching"),
+ ]
+)
+
+MODEL_FOR_TEXT_ENCODING_MAPPING_NAMES = OrderedDict(
+ [
+ ("albert", "AlbertModel"),
+ ("bert", "BertModel"),
+ ("big_bird", "BigBirdModel"),
+ ("clip_text_model", "CLIPTextModel"),
+ ("data2vec-text", "Data2VecTextModel"),
+ ("deberta", "DebertaModel"),
+ ("deberta-v2", "DebertaV2Model"),
+ ("distilbert", "DistilBertModel"),
+ ("electra", "ElectraModel"),
+ ("emu3", "Emu3TextModel"),
+ ("flaubert", "FlaubertModel"),
+ ("ibert", "IBertModel"),
+ ("llama4", "Llama4TextModel"),
+ ("longformer", "LongformerModel"),
+ ("mllama", "MllamaTextModel"),
+ ("mobilebert", "MobileBertModel"),
+ ("mt5", "MT5EncoderModel"),
+ ("nystromformer", "NystromformerModel"),
+ ("reformer", "ReformerModel"),
+ ("rembert", "RemBertModel"),
+ ("roberta", "RobertaModel"),
+ ("roberta-prelayernorm", "RobertaPreLayerNormModel"),
+ ("roc_bert", "RoCBertModel"),
+ ("roformer", "RoFormerModel"),
+ ("squeezebert", "SqueezeBertModel"),
+ ("t5", "T5EncoderModel"),
+ ("t5gemma", "T5GemmaEncoderModel"),
+ ("umt5", "UMT5EncoderModel"),
+ ("xlm", "XLMModel"),
+ ("xlm-roberta", "XLMRobertaModel"),
+ ("xlm-roberta-xl", "XLMRobertaXLModel"),
+ ]
+)
+
+MODEL_FOR_TIME_SERIES_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+ [
+ ("patchtsmixer", "PatchTSMixerForTimeSeriesClassification"),
+ ("patchtst", "PatchTSTForClassification"),
+ ]
+)
+
+MODEL_FOR_TIME_SERIES_REGRESSION_MAPPING_NAMES = OrderedDict(
+ [
+ ("patchtsmixer", "PatchTSMixerForRegression"),
+ ("patchtst", "PatchTSTForRegression"),
+ ]
+)
+
+MODEL_FOR_TIME_SERIES_PREDICTION_MAPPING_NAMES = OrderedDict(
+ [
+ ("timesfm", "TimesFmModelForPrediction"),
+ ("timesfm2_5", "TimesFm2_5ModelForPrediction"),
+ ]
+)
+
+MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES = OrderedDict(
+ [
+ ("swin2sr", "Swin2SRForImageSuperResolution"),
+ ]
+)
+
+MODEL_FOR_AUDIO_TOKENIZATION_NAMES = OrderedDict(
+ [
+ ("dac", "DacModel"),
+ ("higgs_audio_v2_tokenizer", "HiggsAudioV2TokenizerModel"),
+ ("vibevoice_acoustic_tokenizer", "VibeVoiceAcousticTokenizerModel"),
+ ]
+)
+
+MODEL_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_MAPPING_NAMES)
+MODEL_FOR_PRETRAINING_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_PRETRAINING_MAPPING_NAMES)
+MODEL_FOR_CAUSAL_LM_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
+MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING_NAMES
+)
+MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES
+)
+MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES
+)
+MODEL_FOR_IMAGE_SEGMENTATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES
+)
+MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES
+)
+MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING_NAMES
+)
+MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES
+)
+MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES
+)
+MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES
+)
+MODEL_FOR_MULTIMODAL_LM_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES)
+MODEL_FOR_RETRIEVAL_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_RETRIEVAL_MAPPING_NAMES)
+MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES
+)
+MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES
+)
+MODEL_FOR_MASKED_LM_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_MASKED_LM_MAPPING_NAMES)
+MODEL_FOR_IMAGE_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_MAPPING_NAMES)
+MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING_NAMES
+)
+MODEL_FOR_OBJECT_DETECTION_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES)
+MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES
+)
+MODEL_FOR_DEPTH_ESTIMATION_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES)
+MODEL_FOR_TEXT_RECOGNITION_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_TEXT_RECOGNITION_MAPPING_NAMES)
+MODEL_FOR_TABLE_RECOGNITION_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_TABLE_RECOGNITION_MAPPING_NAMES)
+MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES
+)
+MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES
+)
+MODEL_FOR_QUESTION_ANSWERING_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES
+)
+MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES
+)
+MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES
+)
+MODEL_FOR_MULTIPLE_CHOICE_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES)
+MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES
+)
+MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES
+)
+MODEL_FOR_CTC_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_CTC_MAPPING_NAMES)
+MODEL_FOR_TDT_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_TDT_MAPPING_NAMES)
+MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES)
+MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES
+)
+MODEL_FOR_AUDIO_XVECTOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES)
+
+MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES
+)
+
+MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES)
+
+MODEL_FOR_BACKBONE_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_BACKBONE_MAPPING_NAMES)
+
+MODEL_FOR_MASK_GENERATION_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_MASK_GENERATION_MAPPING_NAMES)
+
+MODEL_FOR_KEYPOINT_DETECTION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_KEYPOINT_DETECTION_MAPPING_NAMES
+)
+
+MODEL_FOR_KEYPOINT_MATCHING_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_KEYPOINT_MATCHING_MAPPING_NAMES)
+
+MODEL_FOR_TEXT_ENCODING_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_TEXT_ENCODING_MAPPING_NAMES)
+
+MODEL_FOR_TIME_SERIES_CLASSIFICATION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_TIME_SERIES_CLASSIFICATION_MAPPING_NAMES
+)
+
+MODEL_FOR_TIME_SERIES_REGRESSION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_TIME_SERIES_REGRESSION_MAPPING_NAMES
+)
+
+MODEL_FOR_TIME_SERIES_PREDICTION_MAPPING = _LazyAutoMapping(
+ CONFIG_MAPPING_NAMES, MODEL_FOR_TIME_SERIES_PREDICTION_MAPPING_NAMES
+)
+
+MODEL_FOR_IMAGE_TO_IMAGE_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES)
+
+MODEL_FOR_AUDIO_TOKENIZATION_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_AUDIO_TOKENIZATION_NAMES)
+
+
+class AutoModelForMaskGeneration(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_MASK_GENERATION_MAPPING
+
+
+class AutoModelForKeypointDetection(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_KEYPOINT_DETECTION_MAPPING
+
+
+class AutoModelForKeypointMatching(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_KEYPOINT_MATCHING_MAPPING
+
+
+class AutoModelForTextEncoding(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TEXT_ENCODING_MAPPING
+
+
+class AutoModelForImageToImage(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_IMAGE_TO_IMAGE_MAPPING
+
+
+class AutoModel(_BaseAutoModelClass):
+ _model_mapping = MODEL_MAPPING
+
+
+AutoModel = auto_class_update(AutoModel)
+
+
+class AutoModelForPreTraining(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_PRETRAINING_MAPPING
+
+
+AutoModelForPreTraining = auto_class_update(AutoModelForPreTraining, head_doc="pretraining")
+
+
+class AutoModelForCausalLM(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_CAUSAL_LM_MAPPING
+
+ # override to give better return typehint
+ @classmethod
+ def from_pretrained(
+ cls: type["AutoModelForCausalLM"],
+ pretrained_model_name_or_path: str | os.PathLike[str],
+ *model_args,
+ **kwargs,
+ ) -> "_BaseModelWithGenerate":
+ return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+
+
+AutoModelForCausalLM = auto_class_update(AutoModelForCausalLM, head_doc="causal language modeling")
+
+
+class AutoModelForMaskedLM(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_MASKED_LM_MAPPING
+
+
+AutoModelForMaskedLM = auto_class_update(AutoModelForMaskedLM, head_doc="masked language modeling")
+
+
+class AutoModelForSeq2SeqLM(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
+
+
+AutoModelForSeq2SeqLM = auto_class_update(
+ AutoModelForSeq2SeqLM,
+ head_doc="sequence-to-sequence language modeling",
+ checkpoint_for_example="google-t5/t5-base",
+)
+
+
+class AutoModelForSequenceClassification(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
+
+
+AutoModelForSequenceClassification = auto_class_update(
+ AutoModelForSequenceClassification, head_doc="sequence classification"
+)
+
+
+class AutoModelForQuestionAnswering(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_QUESTION_ANSWERING_MAPPING
+
+
+AutoModelForQuestionAnswering = auto_class_update(AutoModelForQuestionAnswering, head_doc="question answering")
+
+
+class AutoModelForTableQuestionAnswering(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING
+
+
+AutoModelForTableQuestionAnswering = auto_class_update(
+ AutoModelForTableQuestionAnswering,
+ head_doc="table question answering",
+ checkpoint_for_example="google/tapas-base-finetuned-wtq",
+)
+
+
+class AutoModelForVisualQuestionAnswering(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING
+
+
+AutoModelForVisualQuestionAnswering = auto_class_update(
+ AutoModelForVisualQuestionAnswering,
+ head_doc="visual question answering",
+ checkpoint_for_example="dandelin/vilt-b32-finetuned-vqa",
+)
+
+
+class AutoModelForDocumentQuestionAnswering(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING
+
+
+AutoModelForDocumentQuestionAnswering = auto_class_update(
+ AutoModelForDocumentQuestionAnswering,
+ head_doc="document question answering",
+ checkpoint_for_example='impira/layoutlm-document-qa", revision="52e01b3',
+)
+
+
+class AutoModelForTokenClassification(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
+
+
+AutoModelForTokenClassification = auto_class_update(AutoModelForTokenClassification, head_doc="token classification")
+
+
+class AutoModelForMultipleChoice(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_MULTIPLE_CHOICE_MAPPING
+
+
+AutoModelForMultipleChoice = auto_class_update(AutoModelForMultipleChoice, head_doc="multiple choice")
+
+
+class AutoModelForNextSentencePrediction(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING
+
+
+AutoModelForNextSentencePrediction = auto_class_update(
+ AutoModelForNextSentencePrediction, head_doc="next sentence prediction"
+)
+
+
+class AutoModelForImageClassification(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
+
+
+AutoModelForImageClassification = auto_class_update(AutoModelForImageClassification, head_doc="image classification")
+
+
+class AutoModelForZeroShotImageClassification(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
+
+
+AutoModelForZeroShotImageClassification = auto_class_update(
+ AutoModelForZeroShotImageClassification, head_doc="zero-shot image classification"
+)
+
+
+class AutoModelForImageSegmentation(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_IMAGE_SEGMENTATION_MAPPING
+
+
+AutoModelForImageSegmentation = auto_class_update(AutoModelForImageSegmentation, head_doc="image segmentation")
+
+
+class AutoModelForSemanticSegmentation(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING
+
+
+AutoModelForSemanticSegmentation = auto_class_update(
+ AutoModelForSemanticSegmentation, head_doc="semantic segmentation"
+)
+
+
+class AutoModelForTimeSeriesPrediction(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TIME_SERIES_PREDICTION_MAPPING
+
+
+AutoModelForTimeSeriesPrediction = auto_class_update(
+ AutoModelForTimeSeriesPrediction, head_doc="time-series prediction"
+)
+
+
+class AutoModelForUniversalSegmentation(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING
+
+
+AutoModelForUniversalSegmentation = auto_class_update(
+ AutoModelForUniversalSegmentation, head_doc="universal image segmentation"
+)
+
+
+class AutoModelForInstanceSegmentation(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING
+
+
+AutoModelForInstanceSegmentation = auto_class_update(
+ AutoModelForInstanceSegmentation, head_doc="instance segmentation"
+)
+
+
+class AutoModelForObjectDetection(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_OBJECT_DETECTION_MAPPING
+
+
+AutoModelForObjectDetection = auto_class_update(AutoModelForObjectDetection, head_doc="object detection")
+
+
+class AutoModelForZeroShotObjectDetection(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING
+
+
+AutoModelForZeroShotObjectDetection = auto_class_update(
+ AutoModelForZeroShotObjectDetection, head_doc="zero-shot object detection"
+)
+
+
+class AutoModelForDepthEstimation(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_DEPTH_ESTIMATION_MAPPING
+
+
+AutoModelForDepthEstimation = auto_class_update(AutoModelForDepthEstimation, head_doc="depth estimation")
+
+
+class AutoModelForTextRecognition(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TEXT_RECOGNITION_MAPPING
+
+
+AutoModelForTextRecognition = auto_class_update(AutoModelForTextRecognition, head_doc="text recognition")
+
+
+class AutoModelForTableRecognition(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TABLE_RECOGNITION_MAPPING
+
+
+AutoModelForTableRecognition = auto_class_update(AutoModelForTableRecognition, head_doc="table recognition")
+
+
+class AutoModelForVideoClassification(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING
+
+
+AutoModelForVideoClassification = auto_class_update(AutoModelForVideoClassification, head_doc="video classification")
+
+
+class AutoModelForImageTextToText(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING
+
+ # override to give better return typehint
+ @classmethod
+ def from_pretrained(
+ cls: type["AutoModelForImageTextToText"],
+ pretrained_model_name_or_path: str | os.PathLike[str],
+ *model_args,
+ **kwargs,
+ ) -> "_BaseModelWithGenerate":
+ return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+
+
+AutoModelForImageTextToText = auto_class_update(AutoModelForImageTextToText, head_doc="image-text-to-text modeling")
+
+
+class AutoModelForMultimodalLM(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_MULTIMODAL_LM_MAPPING
+
+
+AutoModelForMultimodalLM = auto_class_update(AutoModelForMultimodalLM, head_doc="multimodal generation")
+
+
+class AutoModelForAudioClassification(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING
+
+
+AutoModelForAudioClassification = auto_class_update(AutoModelForAudioClassification, head_doc="audio classification")
+
+
+class AutoModelForCTC(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_CTC_MAPPING
+
+
+AutoModelForCTC = auto_class_update(AutoModelForCTC, head_doc="connectionist temporal classification")
+
+
+class AutoModelForTDT(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TDT_MAPPING
+
+
+AutoModelForTDT = auto_class_update(AutoModelForTDT, head_doc="token-and-duration transducer")
+
+
+class AutoModelForSpeechSeq2Seq(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING
+
+
+AutoModelForSpeechSeq2Seq = auto_class_update(
+ AutoModelForSpeechSeq2Seq, head_doc="sequence-to-sequence speech-to-text modeling"
+)
+
+
+class AutoModelForAudioFrameClassification(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING
+
+
+AutoModelForAudioFrameClassification = auto_class_update(
+ AutoModelForAudioFrameClassification, head_doc="audio frame (token) classification"
+)
+
+
+class AutoModelForAudioXVector(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_AUDIO_XVECTOR_MAPPING
+
+
+class AutoModelForTextToSpectrogram(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING
+
+
+class AutoModelForTextToWaveform(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING
+
+
+class AutoBackbone(_BaseAutoBackboneClass):
+ _model_mapping = MODEL_FOR_BACKBONE_MAPPING
+
+
+AutoModelForAudioXVector = auto_class_update(AutoModelForAudioXVector, head_doc="audio retrieval via x-vector")
+
+
+class AutoModelForMaskedImageModeling(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING
+
+
+AutoModelForMaskedImageModeling = auto_class_update(AutoModelForMaskedImageModeling, head_doc="masked image modeling")
+
+
+class AutoModelForAudioTokenization(_BaseAutoModelClass):
+ _model_mapping = MODEL_FOR_AUDIO_TOKENIZATION_MAPPING
+
+
+AutoModelForAudioTokenization = auto_class_update(
+ AutoModelForAudioTokenization, head_doc="audio tokenization through codebooks"
+)
+
+
+__all__ = [
+ "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_AUDIO_TOKENIZATION_MAPPING",
+ "MODEL_FOR_AUDIO_XVECTOR_MAPPING",
+ "MODEL_FOR_BACKBONE_MAPPING",
+ "MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING",
+ "MODEL_FOR_CAUSAL_LM_MAPPING",
+ "MODEL_FOR_CTC_MAPPING",
+ "MODEL_FOR_TDT_MAPPING",
+ "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING",
+ "MODEL_FOR_DEPTH_ESTIMATION_MAPPING",
+ "MODEL_FOR_TEXT_RECOGNITION_MAPPING",
+ "MODEL_FOR_TABLE_RECOGNITION_MAPPING",
+ "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_IMAGE_MAPPING",
+ "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING",
+ "MODEL_FOR_IMAGE_TO_IMAGE_MAPPING",
+ "MODEL_FOR_KEYPOINT_DETECTION_MAPPING",
+ "MODEL_FOR_KEYPOINT_MATCHING_MAPPING",
+ "MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING",
+ "MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING",
+ "MODEL_FOR_MASKED_LM_MAPPING",
+ "MODEL_FOR_MASK_GENERATION_MAPPING",
+ "MODEL_FOR_MULTIPLE_CHOICE_MAPPING",
+ "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING",
+ "MODEL_FOR_OBJECT_DETECTION_MAPPING",
+ "MODEL_FOR_PRETRAINING_MAPPING",
+ "MODEL_FOR_QUESTION_ANSWERING_MAPPING",
+ "MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING",
+ "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING",
+ "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING",
+ "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING",
+ "MODEL_FOR_TEXT_ENCODING_MAPPING",
+ "MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING",
+ "MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING",
+ "MODEL_FOR_TIME_SERIES_PREDICTION_MAPPING",
+ "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING",
+ "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_RETRIEVAL_MAPPING",
+ "MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING",
+ "MODEL_FOR_MULTIMODAL_LM_MAPPING",
+ "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING",
+ "MODEL_MAPPING",
+ "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING",
+ "MODEL_FOR_TIME_SERIES_CLASSIFICATION_MAPPING",
+ "MODEL_FOR_TIME_SERIES_REGRESSION_MAPPING",
+ "AutoModel",
+ "AutoBackbone",
+ "AutoModelForAudioClassification",
+ "AutoModelForAudioFrameClassification",
+ "AutoModelForAudioTokenization",
+ "AutoModelForAudioXVector",
+ "AutoModelForCausalLM",
+ "AutoModelForCTC",
+ "AutoModelForTDT",
+ "AutoModelForDepthEstimation",
+ "AutoModelForTextRecognition",
+ "AutoModelForTableRecognition",
+ "AutoModelForImageClassification",
+ "AutoModelForImageSegmentation",
+ "AutoModelForImageToImage",
+ "AutoModelForInstanceSegmentation",
+ "AutoModelForKeypointDetection",
+ "AutoModelForKeypointMatching",
+ "AutoModelForMaskGeneration",
+ "AutoModelForTextEncoding",
+ "AutoModelForMaskedImageModeling",
+ "AutoModelForMaskedLM",
+ "AutoModelForMultipleChoice",
+ "AutoModelForMultimodalLM",
+ "AutoModelForNextSentencePrediction",
+ "AutoModelForObjectDetection",
+ "AutoModelForPreTraining",
+ "AutoModelForQuestionAnswering",
+ "AutoModelForSemanticSegmentation",
+ "AutoModelForSeq2SeqLM",
+ "AutoModelForSequenceClassification",
+ "AutoModelForSpeechSeq2Seq",
+ "AutoModelForTableQuestionAnswering",
+ "AutoModelForTextToSpectrogram",
+ "AutoModelForTextToWaveform",
+ "AutoModelForTimeSeriesPrediction",
+ "AutoModelForTokenClassification",
+ "AutoModelForUniversalSegmentation",
+ "AutoModelForVideoClassification",
+ "AutoModelForVisualQuestionAnswering",
+ "AutoModelForDocumentQuestionAnswering",
+ "AutoModelForZeroShotImageClassification",
+ "AutoModelForZeroShotObjectDetection",
+ "AutoModelForImageTextToText",
+]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/processing_auto.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/processing_auto.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b33b31f8ba23d76f6cc99149e546f7e11ced860
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/processing_auto.py
@@ -0,0 +1,474 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""AutoProcessor class."""
+
+import importlib
+import json
+from collections import OrderedDict
+from typing import TYPE_CHECKING
+
+# Build the list of all feature extractors
+from ...configuration_utils import PreTrainedConfig
+from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
+from ...feature_extraction_utils import FeatureExtractionMixin
+from ...image_processing_utils import ImageProcessingMixin
+from ...processing_utils import ProcessorMixin
+from ...tokenization_python import TOKENIZER_CONFIG_FILE
+from ...utils import FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, VIDEO_PROCESSOR_NAME, cached_file, logging
+from ...video_processing_utils import BaseVideoProcessor
+from .auto_factory import _LazyAutoMapping
+from .configuration_auto import (
+ CONFIG_MAPPING_NAMES,
+ AutoConfig,
+ model_type_to_module_name,
+ replace_list_option_in_docstrings,
+)
+from .feature_extraction_auto import AutoFeatureExtractor
+from .image_processing_auto import AutoImageProcessor
+from .tokenization_auto import AutoTokenizer
+from .video_processing_auto import AutoVideoProcessor
+
+
+logger = logging.get_logger(__name__)
+if TYPE_CHECKING:
+ # This significantly improves completion suggestion performance when
+ # the transformers package is used with Microsoft's Pylance language server.
+ PROCESSOR_MAPPING_NAMES: OrderedDict[str, str | None] = OrderedDict()
+else:
+ PROCESSOR_MAPPING_NAMES = OrderedDict(
+ [
+ ("aimv2", "CLIPProcessor"),
+ ("align", "AlignProcessor"),
+ ("altclip", "AltCLIPProcessor"),
+ ("aria", "AriaProcessor"),
+ ("audioflamingo3", "AudioFlamingo3Processor"),
+ ("aya_vision", "AyaVisionProcessor"),
+ ("bark", "BarkProcessor"),
+ ("blip", "BlipProcessor"),
+ ("blip-2", "Blip2Processor"),
+ ("bridgetower", "BridgeTowerProcessor"),
+ ("chameleon", "ChameleonProcessor"),
+ ("chinese_clip", "ChineseCLIPProcessor"),
+ ("clap", "ClapProcessor"),
+ ("clip", "CLIPProcessor"),
+ ("clipseg", "CLIPSegProcessor"),
+ ("clvp", "ClvpProcessor"),
+ ("cohere2_vision", "Cohere2VisionProcessor"),
+ ("cohere_asr", "CohereAsrProcessor"),
+ ("colmodernvbert", "ColModernVBertProcessor"),
+ ("colpali", "ColPaliProcessor"),
+ ("colqwen2", "ColQwen2Processor"),
+ ("deepseek_vl", "DeepseekVLProcessor"),
+ ("deepseek_vl_hybrid", "DeepseekVLHybridProcessor"),
+ ("dia", "DiaProcessor"),
+ ("edgetam", "Sam2Processor"),
+ ("emu3", "Emu3Processor"),
+ ("ernie4_5_vl_moe", "Ernie4_5_VLMoeProcessor"),
+ ("evolla", "EvollaProcessor"),
+ ("exaone4_5", "Exaone4_5_Processor"),
+ ("flava", "FlavaProcessor"),
+ ("florence2", "Florence2Processor"),
+ ("fuyu", "FuyuProcessor"),
+ ("gemma3", "Gemma3Processor"),
+ ("gemma3n", "Gemma3nProcessor"),
+ ("gemma4", "Gemma4Processor"),
+ ("git", "GitProcessor"),
+ ("glm46v", "Glm46VProcessor"),
+ ("glm4v", "Glm4vProcessor"),
+ ("glm4v_moe", "Glm4vProcessor"),
+ ("glm_image", "Glm4vProcessor"),
+ ("glmasr", "GlmAsrProcessor"),
+ ("got_ocr2", "GotOcr2Processor"),
+ ("granite4_vision", "Granite4VisionProcessor"),
+ ("granite_speech", "GraniteSpeechProcessor"),
+ ("granite_speech_plus", "GraniteSpeechProcessor"),
+ ("grounding-dino", "GroundingDinoProcessor"),
+ ("groupvit", "CLIPProcessor"),
+ ("higgs_audio_v2", "HiggsAudioV2Processor"),
+ ("hubert", "Wav2Vec2Processor"),
+ ("idefics", "IdeficsProcessor"),
+ ("idefics2", "Idefics2Processor"),
+ ("idefics3", "Idefics3Processor"),
+ ("instructblip", "InstructBlipProcessor"),
+ ("instructblipvideo", "InstructBlipVideoProcessor"),
+ ("internvl", "InternVLProcessor"),
+ ("janus", "JanusProcessor"),
+ ("kosmos-2", "Kosmos2Processor"),
+ ("kosmos-2.5", "Kosmos2_5Processor"),
+ ("kyutai_speech_to_text", "KyutaiSpeechToTextProcessor"),
+ ("lasr_ctc", "LasrProcessor"),
+ ("lasr_encoder", "LasrProcessor"),
+ ("layoutlmv2", "LayoutLMv2Processor"),
+ ("layoutlmv3", "LayoutLMv3Processor"),
+ ("layoutxlm", "LayoutXLMProcessor"),
+ ("lfm2_vl", "Lfm2VlProcessor"),
+ ("lighton_ocr", "LightOnOcrProcessor"),
+ ("llama4", "Llama4Processor"),
+ ("llava", "LlavaProcessor"),
+ ("llava_next", "LlavaNextProcessor"),
+ ("llava_next_video", "LlavaNextVideoProcessor"),
+ ("llava_onevision", "LlavaOnevisionProcessor"),
+ ("markuplm", "MarkupLMProcessor"),
+ ("metaclip_2", "CLIPProcessor"),
+ ("mgp-str", "MgpstrProcessor"),
+ ("minicpmv4_6", "MiniCPMV4_6Processor"),
+ ("mistral3", "PixtralProcessor"),
+ ("mllama", "MllamaProcessor"),
+ ("mm-grounding-dino", "GroundingDinoProcessor"),
+ ("modernvbert", "Idefics3Processor"),
+ ("moonshine", "Wav2Vec2Processor"),
+ ("moonshine_streaming", "MoonshineStreamingProcessor"),
+ ("musicflamingo", "MusicFlamingoProcessor"),
+ ("omdet-turbo", "OmDetTurboProcessor"),
+ ("oneformer", "OneFormerProcessor"),
+ ("ovis2", "Ovis2Processor"),
+ ("owlv2", "Owlv2Processor"),
+ ("owlvit", "OwlViTProcessor"),
+ ("paddleocr_vl", "PaddleOCRVLProcessor"),
+ ("paligemma", "PaliGemmaProcessor"),
+ ("parakeet_ctc", "ParakeetProcessor"),
+ ("parakeet_tdt", "ParakeetProcessor"),
+ ("perception_lm", "PerceptionLMProcessor"),
+ ("phi4_multimodal", "Phi4MultimodalProcessor"),
+ ("pi0", "PI0Processor"),
+ ("pix2struct", "Pix2StructProcessor"),
+ ("pixtral", "PixtralProcessor"),
+ ("pop2piano", "Pop2PianoProcessor"),
+ ("pp_chart2table", "PPChart2TableProcessor"),
+ ("pp_formulanet", "PPFormulaNetProcessor"),
+ ("qianfan_ocr", "QianfanOCRProcessor"),
+ ("qwen2_5_omni", "Qwen2_5OmniProcessor"),
+ ("qwen2_5_vl", "Qwen2_5_VLProcessor"),
+ ("qwen2_audio", "Qwen2AudioProcessor"),
+ ("qwen2_vl", "Qwen2VLProcessor"),
+ ("qwen3_5", "Qwen3VLProcessor"),
+ ("qwen3_5_moe", "Qwen3VLProcessor"),
+ ("qwen3_omni_moe", "Qwen3OmniMoeProcessor"),
+ ("qwen3_vl", "Qwen3VLProcessor"),
+ ("qwen3_vl_moe", "Qwen3VLProcessor"),
+ ("sam", "SamProcessor"),
+ ("sam2", "Sam2Processor"),
+ ("sam3", "Sam3Processor"),
+ ("sam3_lite_text", "Sam3Processor"),
+ ("sam_hq", "SamHQProcessor"),
+ ("seamless_m4t", "SeamlessM4TProcessor"),
+ ("sew", "Wav2Vec2Processor"),
+ ("sew-d", "Wav2Vec2Processor"),
+ ("shieldgemma2", "ShieldGemma2Processor"),
+ ("siglip", "SiglipProcessor"),
+ ("siglip2", "Siglip2Processor"),
+ ("smolvlm", "SmolVLMProcessor"),
+ ("speech_to_text", "Speech2TextProcessor"),
+ ("speecht5", "SpeechT5Processor"),
+ ("t5gemma2", "Gemma3Processor"),
+ ("t5gemma2_encoder", "Gemma3Processor"),
+ ("trocr", "TrOCRProcessor"),
+ ("tvp", "TvpProcessor"),
+ ("udop", "UdopProcessor"),
+ ("unispeech", "Wav2Vec2Processor"),
+ ("unispeech-sat", "Wav2Vec2Processor"),
+ ("vibevoice_asr", "VibeVoiceAsrProcessor"),
+ ("video_llava", "VideoLlavaProcessor"),
+ ("vilt", "ViltProcessor"),
+ ("vipllava", "LlavaProcessor"),
+ ("vision-text-dual-encoder", "VisionTextDualEncoderProcessor"),
+ ("voxtral", "VoxtralProcessor"),
+ ("voxtral_realtime", "VoxtralRealtimeProcessor"),
+ ("wav2vec2", "Wav2Vec2Processor"),
+ ("wav2vec2-bert", "Wav2Vec2Processor"),
+ ("wav2vec2-conformer", "Wav2Vec2Processor"),
+ ("wavlm", "Wav2Vec2Processor"),
+ ("whisper", "WhisperProcessor"),
+ ("xclip", "XCLIPProcessor"),
+ ]
+ )
+
+PROCESSOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)
+
+
+def processor_class_from_name(class_name: str):
+ for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
+ if class_name in processors:
+ module_name = model_type_to_module_name(module_name)
+
+ module = importlib.import_module(f".{module_name}", "transformers.models")
+ try:
+ return getattr(module, class_name)
+ except AttributeError:
+ continue
+
+ for processor in PROCESSOR_MAPPING._extra_content.values():
+ if getattr(processor, "__name__", None) == class_name:
+ return processor
+
+ # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
+ # init and we return the proper dummy to get an appropriate error message.
+ main_module = importlib.import_module("transformers")
+ if hasattr(main_module, class_name):
+ return getattr(main_module, class_name)
+
+ return None
+
+
+class AutoProcessor:
+ r"""
+ This is a generic processor class that will be instantiated as one of the processor classes of the library when
+ created with the [`AutoProcessor.from_pretrained`] class method.
+
+ This class cannot be instantiated directly using `__init__()` (throws an error).
+ """
+
+ def __init__(self):
+ raise OSError(
+ "AutoProcessor is designed to be instantiated "
+ "using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method."
+ )
+
+ @classmethod
+ @replace_list_option_in_docstrings(PROCESSOR_MAPPING_NAMES)
+ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+ r"""
+ Instantiate one of the processor classes of the library from a pretrained model vocabulary.
+
+ The processor class to instantiate is selected based on the `model_type` property of the config object (either
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible):
+
+ List options
+
+ Params:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a processor files saved using the `save_pretrained()` method,
+ e.g., `./my_model_directory/`.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model feature extractor should be cached if the
+ standard cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the feature extractor files and override the cached versions
+ if they exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ return_unused_kwargs (`bool`, *optional*, defaults to `False`):
+ If `False`, then this function returns just the final feature extractor object. If `True`, then this
+ functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of
+ `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
+ trust_remote_code (`bool`, *optional*, defaults to `False`):
+ Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
+ kwargs (`dict[str, Any]`, *optional*):
+ The values in kwargs of any keys which are feature extractor attributes will be used to override the
+ loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is
+ controlled by the `return_unused_kwargs` keyword parameter.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoProcessor
+
+ >>> # Download processor from huggingface.co and cache.
+ >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")
+
+ >>> # If processor files are in a directory (e.g. processor was saved using *save_pretrained('./test/saved_model/')*)
+ >>> # processor = AutoProcessor.from_pretrained("./test/saved_model/")
+ ```"""
+ config = kwargs.pop("config", None)
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
+ kwargs["_from_auto"] = True
+
+ processor_class = None
+ processor_auto_map = None
+
+ # First, let's see if we have a processor or preprocessor config.
+ # Filter the kwargs for `cached_file`.
+ _hub_valid_kwargs = (
+ "cache_dir",
+ "force_download",
+ "proxies",
+ "token",
+ "revision",
+ "local_files_only",
+ "subfolder",
+ "repo_type",
+ "user_agent",
+ )
+ cached_file_kwargs = {key: kwargs[key] for key in _hub_valid_kwargs if key in kwargs}
+ # We don't want to raise
+ cached_file_kwargs.update(
+ {
+ "_raise_exceptions_for_gated_repo": False,
+ "_raise_exceptions_for_missing_entries": False,
+ "_raise_exceptions_for_connection_errors": False,
+ }
+ )
+
+ # Let's start by checking whether the processor class is saved in a processor config
+ processor_config_file = cached_file(pretrained_model_name_or_path, PROCESSOR_NAME, **cached_file_kwargs)
+ if processor_config_file is not None:
+ config_dict, _ = ProcessorMixin.get_processor_dict(pretrained_model_name_or_path, **kwargs)
+ processor_class = config_dict.get("processor_class")
+ if "AutoProcessor" in config_dict.get("auto_map", {}):
+ processor_auto_map = config_dict["auto_map"]["AutoProcessor"]
+
+ if processor_class is None:
+ # If not found, let's check whether the processor class is saved in an image processor config
+ preprocessor_config_file = cached_file(
+ pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs
+ )
+ if preprocessor_config_file is not None:
+ config_dict, _ = ImageProcessingMixin.get_image_processor_dict(pretrained_model_name_or_path, **kwargs)
+ processor_class = config_dict.get("processor_class", None)
+ if "AutoProcessor" in config_dict.get("auto_map", {}):
+ processor_auto_map = config_dict["auto_map"]["AutoProcessor"]
+
+ # Saved as video processor
+ if preprocessor_config_file is None:
+ preprocessor_config_file = cached_file(
+ pretrained_model_name_or_path, VIDEO_PROCESSOR_NAME, **cached_file_kwargs
+ )
+ if preprocessor_config_file is not None:
+ config_dict, _ = BaseVideoProcessor.get_video_processor_dict(
+ pretrained_model_name_or_path, **kwargs
+ )
+ processor_class = config_dict.get("processor_class", None)
+ if "AutoProcessor" in config_dict.get("auto_map", {}):
+ processor_auto_map = config_dict["auto_map"]["AutoProcessor"]
+ # Saved as feature extractor
+ if preprocessor_config_file is None:
+ preprocessor_config_file = cached_file(
+ pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs
+ )
+ if preprocessor_config_file is not None and processor_class is None:
+ config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(
+ pretrained_model_name_or_path, **kwargs
+ )
+ processor_class = config_dict.get("processor_class", None)
+ if "AutoProcessor" in config_dict.get("auto_map", {}):
+ processor_auto_map = config_dict["auto_map"]["AutoProcessor"]
+
+ if processor_class is None:
+ # Next, let's check whether the processor class is saved in a tokenizer
+ tokenizer_config_file = cached_file(
+ pretrained_model_name_or_path, TOKENIZER_CONFIG_FILE, **cached_file_kwargs
+ )
+ if tokenizer_config_file is not None:
+ with open(tokenizer_config_file, encoding="utf-8") as reader:
+ config_dict = json.load(reader)
+
+ processor_class = config_dict.get("processor_class", None)
+ if "AutoProcessor" in config_dict.get("auto_map", {}):
+ processor_auto_map = config_dict["auto_map"]["AutoProcessor"]
+
+ if processor_class is None:
+ # Last resort: try loading the model config to get processor_class.
+ # This handles cases where processor info is only in config.json (not in any
+ # preprocessor/tokenizer config files). AutoConfig.from_pretrained may raise
+ # ValueError if the model_type is unrecognized or the config is invalid -
+ # we catch and ignore this to allow fallback to AutoTokenizer/AutoImageProcessor.
+ try:
+ if not isinstance(config, PreTrainedConfig):
+ config = AutoConfig.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+
+ processor_class = getattr(config, "processor_class", None)
+ if hasattr(config, "auto_map") and "AutoProcessor" in config.auto_map:
+ processor_auto_map = config.auto_map["AutoProcessor"]
+ except ValueError:
+ # Config loading failed (unrecognized model_type, invalid config, etc.)
+ # Continue to fallback logic below (AutoTokenizer, AutoImageProcessor, etc.)
+ pass
+
+ if processor_class is not None:
+ processor_class = processor_class_from_name(processor_class)
+
+ has_remote_code = processor_auto_map is not None
+ has_local_code = processor_class is not None or type(config) in PROCESSOR_MAPPING
+ explicit_local_code = has_local_code and not (
+ processor_class or PROCESSOR_MAPPING[type(config)]
+ ).__module__.startswith("transformers.")
+ if has_remote_code:
+ if "--" in processor_auto_map:
+ upstream_repo = processor_auto_map.split("--")[0]
+ else:
+ upstream_repo = None
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
+ )
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ processor_class = get_class_from_dynamic_module(
+ processor_auto_map, pretrained_model_name_or_path, **kwargs
+ )
+ _ = kwargs.pop("code_revision", None)
+ processor_class.register_for_auto_class()
+ return processor_class.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+ elif processor_class is not None:
+ return processor_class.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+ # Last try: we use the PROCESSOR_MAPPING.
+ elif type(config) in PROCESSOR_MAPPING:
+ return PROCESSOR_MAPPING[type(config)].from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+ # At this stage, there doesn't seem to be a `Processor` class available for this model.
+ # Let's try the commonly available classes
+ for klass in (AutoTokenizer, AutoImageProcessor, AutoVideoProcessor, AutoFeatureExtractor):
+ try:
+ return klass.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+ except Exception:
+ continue
+
+ raise ValueError(
+ f"Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a "
+ "tokenizer, an image processor, a video processor or a feature extractor for this model. "
+ "Make sure the repository contains the files of at least one of those processing classes."
+ )
+
+ @staticmethod
+ def register(config_class, processor_class, exist_ok=False):
+ """
+ Register a new processor for this class.
+
+ Args:
+ config_class ([`PreTrainedConfig`]):
+ The configuration corresponding to the model to register.
+ processor_class ([`ProcessorMixin`]): The processor to register.
+ """
+ PROCESSOR_MAPPING.register(config_class, processor_class, exist_ok=exist_ok)
+
+
+__all__ = ["PROCESSOR_MAPPING", "AutoProcessor"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py
new file mode 100644
index 0000000000000000000000000000000000000000..cef2e4d5863efe65f89740a49a660df8bfed12a4
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py
@@ -0,0 +1,893 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Auto Tokenizer class."""
+
+import importlib
+import json
+import os
+import sys
+from collections import OrderedDict
+from typing import Any
+
+from transformers.utils.import_utils import is_mistral_common_available
+
+from ...configuration_utils import PreTrainedConfig
+from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
+from ...modeling_gguf_pytorch_utils import load_gguf_checkpoint
+from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
+from ...utils import (
+ extract_commit_hash,
+ is_g2p_en_available,
+ is_sentencepiece_available,
+ is_tokenizers_available,
+ logging,
+)
+from ...utils.hub import cached_file
+from ..encoder_decoder import EncoderDecoderConfig
+from .auto_factory import _LazyAutoMapping
+from .configuration_auto import (
+ CONFIG_MAPPING_NAMES,
+ AutoConfig,
+ config_class_to_model_type,
+ model_type_to_module_name,
+ replace_list_option_in_docstrings,
+)
+
+
+if is_tokenizers_available():
+ from ...tokenization_utils_tokenizers import TokenizersBackend
+else:
+ TokenizersBackend = None
+
+if is_sentencepiece_available():
+ from ...tokenization_utils_sentencepiece import SentencePieceBackend
+else:
+ SentencePieceBackend = None
+
+logger = logging.get_logger(__name__)
+
+# V5: Simplified mapping - single tokenizer class per model type (always prefer tokenizers-based)
+REGISTERED_TOKENIZER_CLASSES: dict[str, type[Any]] = {}
+REGISTERED_FAST_ALIASES: dict[str, type[Any]] = {}
+
+TOKENIZER_MAPPING_NAMES = OrderedDict[str, str | None](
+ [
+ ("aimv2", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("albert", "AlbertTokenizer" if is_tokenizers_available() else None),
+ ("align", "BertTokenizer" if is_tokenizers_available() else None),
+ ("audioflamingo3", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("aya_vision", "CohereTokenizer" if is_tokenizers_available() else None),
+ ("bark", "BertTokenizer" if is_tokenizers_available() else None),
+ ("bart", "RobertaTokenizer" if is_tokenizers_available() else None),
+ ("barthez", "BarthezTokenizer" if is_tokenizers_available() else None),
+ ("bartpho", "BartphoTokenizer"),
+ ("bert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("bert-generation", "BertGenerationTokenizer" if is_sentencepiece_available() else None),
+ ("bert-japanese", "BertJapaneseTokenizer"),
+ ("bertweet", "BertweetTokenizer"),
+ ("big_bird", "BigBirdTokenizer" if is_tokenizers_available() else None),
+ ("bigbird_pegasus", "PegasusTokenizer" if is_tokenizers_available() else None),
+ ("biogpt", "BioGptTokenizer"),
+ ("blenderbot", "BlenderbotTokenizer" if is_tokenizers_available() else None),
+ ("blenderbot-small", "BlenderbotSmallTokenizer"),
+ ("blip", "BertTokenizer" if is_tokenizers_available() else None),
+ ("blip-2", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("bridgetower", "RobertaTokenizer"),
+ ("bros", "BertTokenizer" if is_tokenizers_available() else None),
+ ("byt5", "ByT5Tokenizer"),
+ ("camembert", "CamembertTokenizer" if is_tokenizers_available() else None),
+ ("canine", "CanineTokenizer"),
+ ("chinese_clip", "BertTokenizer" if is_tokenizers_available() else None),
+ ("clap", "RobertaTokenizer"),
+ ("clip", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("clipseg", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("clvp", "ClvpTokenizer"),
+ ("code_llama", "CodeLlamaTokenizer" if is_tokenizers_available() else None),
+ ("codegen", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("cohere", "CohereTokenizer" if is_tokenizers_available() else None),
+ ("cohere2", "CohereTokenizer" if is_tokenizers_available() else None),
+ ("colqwen2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("convbert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("cpm", "CpmTokenizer" if is_tokenizers_available() else None),
+ ("cpmant", "CpmAntTokenizer"),
+ ("ctrl", "CTRLTokenizer"),
+ ("data2vec-audio", "Wav2Vec2CTCTokenizer"),
+ ("data2vec-text", "RobertaTokenizer"),
+ ("dbrx", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("deberta", "DebertaTokenizer" if is_tokenizers_available() else None),
+ ("deberta-v2", "DebertaV2Tokenizer" if is_tokenizers_available() else None),
+ ("dia", "DiaTokenizer"),
+ ("distilbert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("dpr", "DPRQuestionEncoderTokenizer" if is_tokenizers_available() else None),
+ ("electra", "BertTokenizer" if is_tokenizers_available() else None),
+ ("emu3", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("ernie", "BertTokenizer" if is_tokenizers_available() else None),
+ ("esm", "EsmTokenizer"),
+ ("falcon_mamba", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("fastspeech2_conformer", "FastSpeech2ConformerTokenizer" if is_g2p_en_available() else None),
+ ("flaubert", "FlaubertTokenizer"),
+ ("flava", "BertTokenizer" if is_tokenizers_available() else None),
+ ("flex_olmo", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("florence2", "BartTokenizer" if is_tokenizers_available() else None),
+ ("fnet", "FNetTokenizer" if is_tokenizers_available() else None),
+ ("fsmt", "FSMTTokenizer"),
+ ("funnel", "FunnelTokenizer" if is_tokenizers_available() else None),
+ ("gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("gemma2", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("gemma3", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("gemma3_text", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("gemma3n", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("gemma3n_text", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("git", "BertTokenizer" if is_tokenizers_available() else None),
+ ("glm", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("glm4", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("glm4_moe", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("glm4_moe_lite", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("glm4v", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("glm4v_moe", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("glm_image", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("glmasr", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("got_ocr2", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("gpt-sw3", "GPTSw3Tokenizer" if is_sentencepiece_available() else None),
+ ("gpt2", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("gpt_bigcode", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("gpt_neo", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("gpt_neox", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("gpt_neox_japanese", "GPTNeoXJapaneseTokenizer"),
+ ("gptj", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("granite", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("granitemoe", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("granitemoehybrid", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("granitemoeshared", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("grounding-dino", "BertTokenizer" if is_tokenizers_available() else None),
+ ("groupvit", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("herbert", "HerbertTokenizer" if is_tokenizers_available() else None),
+ ("hubert", "Wav2Vec2CTCTokenizer"),
+ ("ibert", "RobertaTokenizer"),
+ ("idefics", "LlamaTokenizer" if is_tokenizers_available() else None),
+ ("idefics2", "LlamaTokenizer" if is_tokenizers_available() else None),
+ ("instructblip", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("instructblipvideo", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("internvl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("jais2", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("jina_embeddings_v3", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
+ ("kosmos-2", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
+ ("lasr_ctc", "LasrTokenizer" if is_tokenizers_available() else None),
+ ("lasr_encoder", "LasrTokenizer" if is_tokenizers_available() else None),
+ ("layoutlm", "BertTokenizer" if is_tokenizers_available() else None),
+ ("layoutlmv2", "LayoutLMv2Tokenizer" if is_tokenizers_available() else None),
+ ("layoutlmv3", "LayoutLMv3Tokenizer" if is_tokenizers_available() else None),
+ ("layoutxlm", "LayoutXLMTokenizer" if is_tokenizers_available() else None),
+ ("led", "LEDTokenizer" if is_tokenizers_available() else None),
+ ("lighton_ocr", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
+ ("lilt", "RobertaTokenizer" if is_tokenizers_available() else None),
+ ("longformer", "RobertaTokenizer" if is_tokenizers_available() else None),
+ ("luke", "LukeTokenizer"),
+ ("lxmert", "LxmertTokenizer" if is_tokenizers_available() else None),
+ ("m2m_100", "M2M100Tokenizer" if is_sentencepiece_available() else None),
+ ("mamba", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("mamba2", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("marian", "MarianTokenizer" if is_sentencepiece_available() else None),
+ ("markuplm", "MarkupLMTokenizer" if is_tokenizers_available() else None),
+ ("mbart", "MBartTokenizer" if is_tokenizers_available() else None),
+ ("mbart50", "MBart50Tokenizer" if is_tokenizers_available() else None),
+ ("mega", "RobertaTokenizer"),
+ ("megatron-bert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("metaclip_2", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
+ ("mgp-str", "MgpstrTokenizer"),
+ ("minicpmv4_6", "TokenizersBackend" if is_tokenizers_available() else None),
+ (
+ "ministral",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ (
+ "ministral3",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ (
+ "mistral",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ (
+ "mistral3",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ (
+ "mixtral",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ ("mluke", "MLukeTokenizer" if is_sentencepiece_available() else None),
+ ("mm-grounding-dino", "BertTokenizer" if is_tokenizers_available() else None),
+ ("mobilebert", "MobileBertTokenizer" if is_tokenizers_available() else None),
+ ("mpnet", "MPNetTokenizer" if is_tokenizers_available() else None),
+ ("mpt", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("mra", "RobertaTokenizer"),
+ ("mt5", "T5Tokenizer" if is_tokenizers_available() else None),
+ ("musicgen", "T5Tokenizer" if is_tokenizers_available() else None),
+ ("musicgen_melody", "T5Tokenizer" if is_tokenizers_available() else None),
+ ("mvp", "MvpTokenizer" if is_tokenizers_available() else None),
+ ("myt5", "MyT5Tokenizer"),
+ ("nezha", "BertTokenizer" if is_tokenizers_available() else None),
+ ("nllb", "NllbTokenizer" if is_tokenizers_available() else None),
+ ("nllb-moe", "NllbTokenizer" if is_tokenizers_available() else None),
+ ("nomic_bert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("nougat", "NougatTokenizer" if is_tokenizers_available() else None),
+ ("nystromformer", "AlbertTokenizer" if is_tokenizers_available() else None),
+ ("olmo", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("olmo2", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("olmo3", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("olmo_hybrid", "TokenizersBackend" if is_tokenizers_available() else None),
+ ("olmoe", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("omdet-turbo", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("oneformer", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("openai-gpt", "OpenAIGPTTokenizer" if is_tokenizers_available() else None),
+ ("opt", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("ovis2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("owlv2", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("owlvit", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("parakeet_ctc", "ParakeetTokenizer" if is_tokenizers_available() else None),
+ ("parakeet_tdt", "ParakeetTokenizer" if is_tokenizers_available() else None),
+ ("pegasus", "PegasusTokenizer" if is_tokenizers_available() else None),
+ ("pegasus_x", "PegasusTokenizer" if is_tokenizers_available() else None),
+ ("perceiver", "PerceiverTokenizer"),
+ ("phi", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("phobert", "PhobertTokenizer"),
+ ("pix2struct", "T5Tokenizer" if is_tokenizers_available() else None),
+ (
+ "pixtral",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ ("plbart", "PLBartTokenizer" if is_tokenizers_available() else None),
+ ("pp_formulanet", "NougatTokenizer" if is_tokenizers_available() else None),
+ ("prophetnet", "ProphetNetTokenizer"),
+ ("qdqbert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("qianfan_ocr", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen2_5_omni", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen2_5_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen2_audio", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen2_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen2_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3_5", "Qwen3_5Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3_5_moe", "Qwen3_5Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3_next", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3_omni_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("qwen3_vl_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+ ("rag", "RagTokenizer"),
+ ("realm", "BertTokenizer" if is_tokenizers_available() else None),
+ ("recurrent_gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("reformer", "ReformerTokenizer" if is_tokenizers_available() else None),
+ ("rembert", "RemBertTokenizer" if is_tokenizers_available() else None),
+ ("retribert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("roberta", "RobertaTokenizer"),
+ ("roberta-prelayernorm", "RobertaTokenizer"),
+ ("roc_bert", "RoCBertTokenizer"),
+ ("roformer", "RoFormerTokenizer" if is_tokenizers_available() else None),
+ ("rwkv", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("sam3", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("sam3_video", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("seamless_m4t", "SeamlessM4TTokenizer" if is_tokenizers_available() else None),
+ ("seamless_m4t_v2", "SeamlessM4TTokenizer" if is_tokenizers_available() else None),
+ ("shieldgemma2", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("siglip", "SiglipTokenizer" if is_sentencepiece_available() else None),
+ ("siglip2", "Siglip2Tokenizer" if is_tokenizers_available() else None),
+ ("speech_to_text", "Speech2TextTokenizer" if is_sentencepiece_available() else None),
+ ("speecht5", "SpeechT5Tokenizer" if is_sentencepiece_available() else None),
+ ("splinter", "SplinterTokenizer"),
+ ("squeezebert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("stablelm", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("starcoder2", "GPT2Tokenizer" if is_tokenizers_available() else None),
+ ("switch_transformers", "T5Tokenizer" if is_tokenizers_available() else None),
+ ("t5", "T5Tokenizer" if is_tokenizers_available() else None),
+ ("t5gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
+ ("tapas", "TapasTokenizer"),
+ ("trocr", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
+ ("tvp", "BertTokenizer" if is_tokenizers_available() else None),
+ ("udop", "UdopTokenizer" if is_tokenizers_available() else None),
+ ("umt5", "T5Tokenizer" if is_tokenizers_available() else None),
+ ("unispeech", "Wav2Vec2CTCTokenizer"),
+ ("unispeech-sat", "Wav2Vec2CTCTokenizer"),
+ ("vilt", "BertTokenizer" if is_tokenizers_available() else None),
+ ("visual_bert", "BertTokenizer" if is_tokenizers_available() else None),
+ ("vits", "VitsTokenizer"),
+ (
+ "voxtral",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ (
+ "voxtral_realtime",
+ "MistralCommonBackend"
+ if is_mistral_common_available()
+ else ("TokenizersBackend" if is_tokenizers_available() else None),
+ ),
+ ("wav2vec2", "Wav2Vec2CTCTokenizer"),
+ ("wav2vec2-bert", "Wav2Vec2CTCTokenizer"),
+ ("wav2vec2-conformer", "Wav2Vec2CTCTokenizer"),
+ ("wav2vec2_phoneme", "Wav2Vec2PhonemeCTCTokenizer"),
+ ("whisper", "WhisperTokenizer" if is_tokenizers_available() else None),
+ ("xclip", "CLIPTokenizer" if is_tokenizers_available() else None),
+ ("xglm", "XGLMTokenizer" if is_tokenizers_available() else None),
+ ("xlm", "XLMTokenizer"),
+ ("xlm-roberta", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
+ ("xlm-roberta-xl", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
+ ("xlnet", "XLNetTokenizer" if is_tokenizers_available() else None),
+ ("xlstm", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+ ("xmod", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
+ ("yoso", "AlbertTokenizer" if is_tokenizers_available() else None),
+ ]
+)
+
+# Models with incorrect tokenizer_class in their Hub tokenizer_config.json files.
+# These models will be forced to use TokenizersBackend.
+MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS: set[str] = {
+ "arctic",
+ "chameleon",
+ "chatlm",
+ "deepseek_v2",
+ "deepseek_v3",
+ "deepseek_v4",
+ "deepseek_vl",
+ "deepseek_vl_hybrid",
+ "deepseek_vl_v2",
+ "deepseek_ocr",
+ "deepseek_ocr2",
+ "fuyu",
+ "h2ovl_chat",
+ "hyperclovax_vlm",
+ "internlm2",
+ "internvl_chat",
+ "jamba",
+ "janus",
+ "llava",
+ "llava_next",
+ "minicpmv",
+ "minimax_m2",
+ "modernbert",
+ "molmo",
+ "molmo2",
+ "nemotron",
+ "nvfp4",
+ "opencua",
+ "openvla",
+ "phi3",
+ "phi3_v",
+ "phimoe",
+ "qwen2",
+ "step3p5",
+ "step3_vl",
+ "vipllava",
+ "cohere_asr",
+}
+
+for model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS:
+ if model_type not in TOKENIZER_MAPPING_NAMES:
+ TOKENIZER_MAPPING_NAMES[model_type] = "TokenizersBackend" if is_tokenizers_available() else None
+
+TOKENIZER_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, TOKENIZER_MAPPING_NAMES)
+
+CONFIG_TO_TYPE = {v: k for k, v in CONFIG_MAPPING_NAMES.items()}
+
+
+def load_vocab(vocab_file):
+ """Loads a vocabulary file into a dictionary."""
+ with open(vocab_file, "r", encoding="utf-8") as reader:
+ return json.load(reader)
+
+
+def load_merges(merges_file):
+ """Loads a merges file into a list."""
+ merges = []
+ with open(merges_file, "r", encoding="utf-8") as reader:
+ for line in reader:
+ line = line.strip()
+ if line and not line.startswith("#"):
+ merges.append(tuple(line.split()))
+ return merges
+
+
+def tokenizer_class_from_name(class_name: str) -> type[Any] | None:
+ # Bloom tokenizer classes were removed but should map to the fast backend for BC
+ if class_name in {"BloomTokenizer", "BloomTokenizerFast"}:
+ return TokenizersBackend
+
+ if class_name in REGISTERED_FAST_ALIASES:
+ return REGISTERED_FAST_ALIASES[class_name]
+
+ if class_name in REGISTERED_TOKENIZER_CLASSES:
+ return REGISTERED_TOKENIZER_CLASSES[class_name]
+
+ if class_name == "TokenizersBackend":
+ return TokenizersBackend
+
+ # V5: TOKENIZER_MAPPING_NAMES now maps to single strings, not tuples
+ for module_name, tokenizer_class in TOKENIZER_MAPPING_NAMES.items():
+ if tokenizer_class == class_name:
+ module_name = model_type_to_module_name(module_name)
+ if (
+ module_name in ["mistral", "mistral3", "mixtral", "ministral", "ministral3", "pixtral", "voxtral"]
+ and class_name == "MistralCommonBackend"
+ ):
+ module = importlib.import_module(".tokenization_mistral_common", "transformers")
+ else:
+ module = importlib.import_module(f".{module_name}", "transformers.models")
+ try:
+ result = getattr(module, class_name)
+ # BC v5: expose XxxFast alias and tokenization_*_fast submodule for pre-v5 remote code.
+ if (submod := getattr(result, "__module__", None)) and submod in sys.modules:
+ base_mod = sys.modules[submod]
+ setattr(base_mod, result.__name__ + "Fast", result)
+ sys.modules.setdefault(submod + "_fast", base_mod)
+ return result
+ except AttributeError:
+ continue
+
+ for tokenizer in TOKENIZER_MAPPING._extra_content.values():
+ if getattr(tokenizer, "__name__", None) == class_name:
+ return tokenizer
+
+ # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
+ # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
+ # init and we return the proper dummy to get an appropriate error message.
+ main_module = importlib.import_module("transformers")
+ if hasattr(main_module, class_name):
+ return getattr(main_module, class_name)
+
+ # BC v5: If a XxxFast class is not found, retry without 'Fast' for tokenizers saved pre-v5.
+ if class_name.endswith("Fast"):
+ return tokenizer_class_from_name(class_name[:-4])
+
+ return None
+
+
+def get_tokenizer_config(
+ pretrained_model_name_or_path: str | os.PathLike[str],
+ cache_dir: str | os.PathLike[str] | None = None,
+ force_download: bool = False,
+ proxies: dict[str, str] | None = None,
+ token: bool | str | None = None,
+ revision: str | None = None,
+ local_files_only: bool = False,
+ subfolder: str = "",
+ **kwargs,
+) -> dict[str, Any]:
+ """
+ Loads the tokenizer configuration from a pretrained model tokenizer configuration.
+
+ Args:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a configuration file saved using the
+ [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
+
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
+ cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the configuration files and override the cached versions if they
+ exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ local_files_only (`bool`, *optional*, defaults to `False`):
+ If `True`, will only try to load the tokenizer configuration from local files.
+ subfolder (`str`, *optional*, defaults to `""`):
+ In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can
+ specify the folder name here.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Returns:
+ `dict`: The configuration of the tokenizer.
+
+ Examples:
+
+ ```python
+ # Download configuration from huggingface.co and cache.
+ tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased")
+ # This model does not have a tokenizer config so the result will be an empty dict.
+ tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base")
+
+ # Save a pretrained tokenizer locally and you can reload its config
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased")
+ tokenizer.save_pretrained("tokenizer-test")
+ tokenizer_config = get_tokenizer_config("tokenizer-test")
+ ```"""
+ commit_hash = kwargs.get("_commit_hash")
+ resolved_config_file = cached_file(
+ pretrained_model_name_or_path,
+ TOKENIZER_CONFIG_FILE,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ subfolder=subfolder,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ _commit_hash=commit_hash,
+ )
+ if resolved_config_file is None:
+ logger.info("Could not locate the tokenizer configuration file, will try to use the model config instead.")
+ return {}
+ commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
+
+ with open(resolved_config_file, encoding="utf-8") as reader:
+ result = json.load(reader)
+ result["_commit_hash"] = commit_hash
+ return result
+
+
+class AutoTokenizer:
+ r"""
+ This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when
+ created with the [`AutoTokenizer.from_pretrained`] class method.
+
+ This class cannot be instantiated directly using `__init__()` (throws an error).
+ """
+
+ def __init__(self):
+ raise OSError(
+ "AutoTokenizer is designed to be instantiated "
+ "using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method."
+ )
+
+ @classmethod
+ @replace_list_option_in_docstrings(TOKENIZER_MAPPING_NAMES)
+ def from_pretrained(
+ cls, pretrained_model_name_or_path, *inputs, **kwargs
+ ) -> TokenizersBackend | SentencePieceBackend:
+ r"""
+ Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary.
+
+ The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
+ falling back to using pattern matching on `pretrained_model_name_or_path`:
+
+ List options
+
+ Params:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ Can be either:
+
+ - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
+ - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
+ using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
+ - a path to a single saved vocabulary file if and only if the tokenizer only requires a
+ single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not
+ applicable to all derived classes)
+ inputs (additional positional arguments, *optional*):
+ Will be passed along to the Tokenizer `__init__()` method.
+ config ([`PreTrainedConfig`], *optional*)
+ The configuration object used to determine the tokenizer class to instantiate.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model configuration should be cached if the
+ standard cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force the (re-)download the model weights and configuration files and override the
+ cached versions if they exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ subfolder (`str`, *optional*):
+ In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for
+ facebook/rag-token-base), specify it here.
+ tokenizer_type (`str`, *optional*):
+ Tokenizer type to be loaded.
+ backend (`str`, *optional*, defaults to `"tokenizers"`):
+ Backend to use for tokenization. Valid options are:
+ - `"tokenizers"`: Use the HuggingFace tokenizers library backend (default)
+ - `"sentencepiece"`: Use the SentencePiece backend
+ trust_remote_code (`bool`, *optional*, defaults to `False`):
+ Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
+ kwargs (additional keyword arguments, *optional*):
+ Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like
+ `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
+ `additional_special_tokens`. See parameters in the `__init__()` for more details.
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer
+
+ >>> # Download vocabulary from huggingface.co and cache.
+ >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")
+
+ >>> # Download vocabulary from huggingface.co (user-uploaded) and cache.
+ >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased")
+
+ >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*)
+ >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/")
+
+ >>> # Download vocabulary from huggingface.co and define model-specific arguments
+ >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True)
+
+ >>> # Explicitly use the tokenizers backend
+ >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="tokenizers")
+
+ >>> # Explicitly use the sentencepiece backend
+ >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="sentencepiece")
+ ```"""
+ config = kwargs.pop("config", None)
+ kwargs["_from_auto"] = True
+
+ # V5: Always use fast tokenizers, ignore use_fast parameter
+ _ = kwargs.pop("use_fast", None)
+ tokenizer_type = kwargs.pop("tokenizer_type", None)
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
+ gguf_file = kwargs.get("gguf_file")
+
+ # First, let's see whether the tokenizer_type is passed so that we can leverage it
+ if tokenizer_type is not None:
+ tokenizer_class_name = TOKENIZER_MAPPING_NAMES.get(tokenizer_type, None)
+
+ if tokenizer_class_name is None:
+ raise ValueError(
+ f"Passed `tokenizer_type` {tokenizer_type} does not exist. `tokenizer_type` should be one of "
+ f"{', '.join(c for c in TOKENIZER_MAPPING_NAMES)}."
+ )
+
+ tokenizer_class = tokenizer_class_from_name(tokenizer_class_name)
+
+ if tokenizer_class is None:
+ raise ValueError(f"Tokenizer class {tokenizer_class_name} is not currently imported.")
+
+ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ if gguf_file:
+ gguf_path = cached_file(pretrained_model_name_or_path, gguf_file, **kwargs)
+ config_dict = load_gguf_checkpoint(gguf_path, return_tensors=False)["config"]
+ config = AutoConfig.for_model(**config_dict)
+ elif config is None:
+ try:
+ config = AutoConfig.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+ except (ValueError, OSError):
+ config = PreTrainedConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+ config_model_type = config.model_type
+
+ # Next, let's try to use the tokenizer_config file to get the tokenizer class.
+ tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
+ tokenizer_config_class = tokenizer_config.get("tokenizer_class", None)
+
+ # Check for auto_map early to handle dynamic tokenizers properly
+ tokenizer_auto_map = None
+ if "auto_map" in tokenizer_config:
+ if isinstance(tokenizer_config["auto_map"], (tuple, list)):
+ # Legacy format for dynamic tokenizers
+ tokenizer_auto_map = tokenizer_config["auto_map"]
+ else:
+ tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)
+
+ # if there is a config, we can check that the tokenizer class != than model class.
+ # Use the config class if it's a specialized tokenizer, otherwise fall back to TokenizersBackend.
+ if (
+ tokenizer_auto_map is None
+ and tokenizer_config_class is not None
+ and config_model_type is not None
+ and config_model_type != ""
+ and TOKENIZER_MAPPING_NAMES.get(config_model_type) is not None
+ and (TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast"))
+ != (tokenizer_config_class.removesuffix("Fast"))
+ ):
+ registered_class_name = TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast")
+ if registered_class_name not in ("TokenizersBackend", "PythonBackend", "PreTrainedTokenizerFast"):
+ # If the hub class is known incorrect for this model type, use the registered class; otherwise trust the hub.
+ class_name = (
+ registered_class_name
+ if config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS
+ else tokenizer_config_class
+ )
+ tokenizer_class = tokenizer_class_from_name(class_name)
+ if tokenizer_class is not None and tokenizer_class.__name__ not in (
+ "TokenizersBackend",
+ "PythonBackend",
+ "PreTrainedTokenizerFast",
+ ):
+ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ if TokenizersBackend is not None:
+ return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ raise ValueError(
+ f"Tokenizer class '{tokenizer_config_class}' specified in the tokenizer config was not found. "
+ f"The tokenizer may need to be converted or re-saved."
+ )
+
+ if "_commit_hash" in tokenizer_config:
+ kwargs["_commit_hash"] = tokenizer_config["_commit_hash"]
+
+ if tokenizer_config_class and tokenizer_config_class.endswith("Fast"):
+ tokenizer_config_class = tokenizer_config_class[:-4]
+
+ has_remote_code = tokenizer_auto_map is not None
+ has_local_code = type(config) in TOKENIZER_MAPPING or (
+ tokenizer_config_class is not None
+ and (
+ tokenizer_class_from_name(tokenizer_config_class) is not None
+ or tokenizer_class_from_name(tokenizer_config_class + "Fast") is not None
+ )
+ )
+ explicit_local_code = (
+ has_local_code
+ and type(config) not in TOKENIZER_MAPPING
+ and (
+ tokenizer_config_class is not None
+ and not (
+ tokenizer_class_from_name(tokenizer_config_class)
+ or tokenizer_class_from_name(tokenizer_config_class + "Fast")
+ ).__module__.startswith("transformers.")
+ )
+ )
+ # V5: Skip remote tokenizer for custom models with incorrect hub tokenizer class
+ if has_remote_code and config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS:
+ has_remote_code = False
+ tokenizer_auto_map = None
+
+ if has_remote_code:
+ # V5: Always prefer fast tokenizer (index 1), fallback to slow (index 0)
+ if tokenizer_auto_map[1] is not None:
+ class_ref = tokenizer_auto_map[1]
+ else:
+ class_ref = tokenizer_auto_map[0]
+ if "--" in class_ref:
+ upstream_repo = class_ref.split("--")[0]
+ else:
+ upstream_repo = None
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
+ )
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ # BC v5: register *Fast aliases before remote code loads.
+ if tokenizer_config_class:
+ tokenizer_class_from_name(tokenizer_config_class.removesuffix("Fast"))
+ tokenizer_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
+ _ = kwargs.pop("code_revision", None)
+ tokenizer_class.register_for_auto_class()
+ return tokenizer_class.from_pretrained(
+ pretrained_model_name_or_path, *inputs, trust_remote_code=trust_remote_code, **kwargs
+ )
+ elif tokenizer_config_class is not None:
+ tokenizer_class_candidate = tokenizer_config_class
+ tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
+ if tokenizer_class is None and not tokenizer_class_candidate.endswith("Fast"):
+ tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate + "Fast")
+ if tokenizer_class is not None and tokenizer_class.__name__ == "PythonBackend":
+ tokenizer_class = TokenizersBackend
+ # Fallback to TokenizersBackend if the class wasn't found
+ if tokenizer_class is None:
+ tokenizer_class = TokenizersBackend
+
+ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+ elif getattr(config, "tokenizer_class", None):
+ _class = config.tokenizer_class
+ if "PreTrainedTokenizerFast" not in _class and _class.endswith("Fast"):
+ _class = _class[:-4]
+ tokenizer_class = tokenizer_class_from_name(_class)
+ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ # Otherwise we have to be creative.
+ # if model is an encoder decoder, the encoder tokenizer class is used by default
+ if isinstance(config, EncoderDecoderConfig):
+ if type(config.decoder) is not type(config.encoder):
+ logger.warning(
+ f"The encoder model config class: {config.encoder.__class__} is different from the decoder model "
+ f"config class: {config.decoder.__class__}. It is not recommended to use the "
+ "`AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder "
+ "specific tokenizer classes."
+ )
+ config = config.encoder
+
+ model_type = config_class_to_model_type(type(config).__name__) or getattr(config, "model_type", None)
+ if model_type is not None:
+ tokenizer_class = TOKENIZER_MAPPING.get(type(config), TokenizersBackend)
+ if tokenizer_class is not None:
+ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ # Fallback: try tokenizer_class from tokenizer_config.json
+ tokenizer_config_class = tokenizer_config.get("tokenizer_class", None)
+ if tokenizer_config_class is not None:
+ if tokenizer_config_class != "TokenizersBackend" and tokenizer_config_class.endswith("Fast"):
+ tokenizer_config_class = tokenizer_config_class[:-4]
+ tokenizer_class = tokenizer_class_from_name(tokenizer_config_class)
+ if tokenizer_class is None and not tokenizer_config_class.endswith("Fast"):
+ tokenizer_class = tokenizer_class_from_name(tokenizer_config_class + "Fast")
+ if tokenizer_class is not None and tokenizer_class.__name__ == "PythonBackend":
+ tokenizer_class = TokenizersBackend
+ if tokenizer_class is None:
+ tokenizer_class = TokenizersBackend
+ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ raise ValueError(
+ f"Unrecognized configuration class {config.__class__} to build an AutoTokenizer.\n"
+ f"Model type should be one of {', '.join(c.__name__ for c in TOKENIZER_MAPPING)}."
+ )
+
+ @staticmethod
+ def register(
+ config_class, tokenizer_class=None, slow_tokenizer_class=None, fast_tokenizer_class=None, exist_ok=False
+ ):
+ """
+ Register a new tokenizer in this mapping.
+
+ Args:
+ config_class ([`PreTrainedConfig`]):
+ The configuration corresponding to the model to register.
+ tokenizer_class: The tokenizer class to register (V5 - preferred parameter).
+ slow_tokenizer_class: (Deprecated) The slow tokenizer to register.
+ fast_tokenizer_class: (Deprecated) The fast tokenizer to register.
+ """
+ if tokenizer_class is None:
+ # Legacy: prefer fast over slow
+ if fast_tokenizer_class is not None:
+ tokenizer_class = fast_tokenizer_class
+ elif slow_tokenizer_class is not None:
+ tokenizer_class = slow_tokenizer_class
+ else:
+ raise ValueError("You need to pass a `tokenizer_class`")
+
+ for candidate in (slow_tokenizer_class, fast_tokenizer_class, tokenizer_class):
+ if candidate is not None:
+ REGISTERED_TOKENIZER_CLASSES[candidate.__name__] = candidate
+
+ if slow_tokenizer_class is not None and fast_tokenizer_class is not None:
+ REGISTERED_FAST_ALIASES[slow_tokenizer_class.__name__] = fast_tokenizer_class
+
+ TOKENIZER_MAPPING.register(config_class, tokenizer_class, exist_ok=exist_ok)
+
+
+__all__ = ["TOKENIZER_MAPPING", "AutoTokenizer"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/video_processing_auto.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/video_processing_auto.py
new file mode 100644
index 0000000000000000000000000000000000000000..5bf15a9d6f87fbb873e51132581e62d724218b77
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/auto/video_processing_auto.py
@@ -0,0 +1,408 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""AutoVideoProcessor class."""
+
+import importlib
+import os
+from collections import OrderedDict
+from typing import TYPE_CHECKING
+
+# Build the list of all video processors
+from ...configuration_utils import PreTrainedConfig
+from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
+from ...utils import (
+ CONFIG_NAME,
+ IMAGE_PROCESSOR_NAME,
+ PROCESSOR_NAME,
+ VIDEO_PROCESSOR_NAME,
+ cached_file,
+ is_torchvision_available,
+ logging,
+ safe_load_json_file,
+)
+from ...utils.import_utils import requires
+from ...video_processing_utils import BaseVideoProcessor
+from .auto_factory import _LazyAutoMapping
+from .auto_mappings import VIDEO_PROCESSOR_MAPPING_NAMES
+from .configuration_auto import (
+ CONFIG_MAPPING_NAMES,
+ AutoConfig,
+ model_type_to_module_name,
+ replace_list_option_in_docstrings,
+)
+
+
+logger = logging.get_logger(__name__)
+
+
+if TYPE_CHECKING:
+ # This significantly improves completion suggestion performance when
+ # the transformers package is used with Microsoft's Pylance language server.
+ VIDEO_PROCESSOR_MAPPING_NAMES: OrderedDict[str, tuple[str | None, str | None]] = OrderedDict()
+else:
+ # Merge non-standard mapping names with auto-inferred `VIDEO_PROCESSOR_MAPPING_NAMES`
+ MISSING_VIDEO_PROCESSOR_MAPPING_NAMES = OrderedDict(
+ [
+ ("exaone4_5", "Qwen2VLVideoProcessor"),
+ ("instructblip", "InstructBlipVideoVideoProcessor"),
+ ("pe_audio_video", "PeVideoVideoProcessor"),
+ ("qwen2_5_omni", "Qwen2VLVideoProcessor"),
+ ("qwen2_5_vl", "Qwen2VLVideoProcessor"),
+ ("qwen3_5", "Qwen3VLVideoProcessor"),
+ ("qwen3_5_moe", "Qwen3VLVideoProcessor"),
+ ("qwen3_omni_moe", "Qwen2VLVideoProcessor"),
+ ("qwen3_vl_moe", "Qwen3VLVideoProcessor"),
+ ]
+ )
+ VIDEO_PROCESSOR_MAPPING_NAMES.update(MISSING_VIDEO_PROCESSOR_MAPPING_NAMES)
+
+for model_type, video_processors in VIDEO_PROCESSOR_MAPPING_NAMES.items():
+ fast_video_processor_class = video_processors
+
+ # If the torchvision is not available, we set it to None
+ if not is_torchvision_available():
+ fast_video_processor_class = None
+
+ VIDEO_PROCESSOR_MAPPING_NAMES[model_type] = fast_video_processor_class
+
+VIDEO_PROCESSOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, VIDEO_PROCESSOR_MAPPING_NAMES)
+
+
+def video_processor_class_from_name(class_name: str):
+ for module_name, extractor in VIDEO_PROCESSOR_MAPPING_NAMES.items():
+ if class_name == extractor:
+ module_name = model_type_to_module_name(module_name)
+
+ module = importlib.import_module(f".{module_name}", "transformers.models")
+ try:
+ return getattr(module, class_name)
+ except AttributeError:
+ continue
+
+ for extractor in VIDEO_PROCESSOR_MAPPING._extra_content.values():
+ if getattr(extractor, "__name__", None) == class_name:
+ return extractor
+
+ # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
+ # init and we return the proper dummy to get an appropriate error message.
+ main_module = importlib.import_module("transformers")
+ if hasattr(main_module, class_name):
+ return getattr(main_module, class_name)
+
+ return None
+
+
+def get_video_processor_config(
+ pretrained_model_name_or_path: str | os.PathLike,
+ cache_dir: str | os.PathLike | None = None,
+ force_download: bool = False,
+ proxies: dict[str, str] | None = None,
+ token: bool | str | None = None,
+ revision: str | None = None,
+ local_files_only: bool = False,
+ **kwargs,
+):
+ """
+ Loads the video processor configuration from a pretrained model video processor configuration.
+
+ Args:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a configuration file saved using the
+ [`~BaseVideoProcessor.save_pretrained`] method, e.g., `./my_model_directory/`.
+
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
+ cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the configuration files and override the cached versions if they
+ exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ local_files_only (`bool`, *optional*, defaults to `False`):
+ If `True`, will only try to load the video processor configuration from local files.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Returns:
+ `Dict`: The configuration of the video processor.
+
+ Examples:
+
+ ```python
+ # Download configuration from huggingface.co and cache.
+ video_processor_config = get_video_processor_config("llava-hf/llava-onevision-qwen2-0.5b-ov-hf")
+ # This model does not have a video processor config so the result will be an empty dict.
+ video_processor_config = get_video_processor_config("FacebookAI/xlm-roberta-base")
+
+ # Save a pretrained video processor locally and you can reload its config
+ from transformers import AutoVideoProcessor
+
+ video_processor = AutoVideoProcessor.from_pretrained("llava-hf/llava-onevision-qwen2-0.5b-ov-hf")
+ video_processor.save_pretrained("video-processor-test")
+ video_processor = get_video_processor_config("video-processor-test")
+ ```"""
+ # Load with a priority given to the nested processor config, if available in repo
+ resolved_processor_file = cached_file(
+ pretrained_model_name_or_path,
+ filename=PROCESSOR_NAME,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ )
+ resolved_video_processor_files = [
+ resolved_file
+ for filename in [VIDEO_PROCESSOR_NAME, IMAGE_PROCESSOR_NAME]
+ if (
+ resolved_file := cached_file(
+ pretrained_model_name_or_path,
+ filename=filename,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ )
+ )
+ is not None
+ ]
+ resolved_video_processor_file = resolved_video_processor_files[0] if resolved_video_processor_files else None
+
+ # An empty list if none of the possible files is found in the repo
+ if not resolved_video_processor_file and not resolved_processor_file:
+ logger.info("Could not locate the video processor configuration file.")
+ return {}
+
+ # Load video_processor dict. Priority goes as (nested config if found -> video processor config -> image processor config)
+ # We are downloading both configs because almost all models have a `processor_config.json` but
+ # not all of these are nested. We need to check if it was saved recebtly as nested or if it is legacy style
+ video_processor_dict = {}
+ if resolved_processor_file is not None:
+ processor_dict = safe_load_json_file(resolved_processor_file)
+ if "video_processor" in processor_dict:
+ video_processor_dict = processor_dict["video_processor"]
+
+ if resolved_video_processor_file is not None and video_processor_dict is None:
+ video_processor_dict = safe_load_json_file(resolved_video_processor_file)
+
+ return video_processor_dict
+
+
+@requires(backends=("vision", "torchvision"))
+class AutoVideoProcessor:
+ r"""
+ This is a generic video processor class that will be instantiated as one of the video processor classes of the
+ library when created with the [`AutoVideoProcessor.from_pretrained`] class method.
+
+ This class cannot be instantiated directly using `__init__()` (throws an error).
+ """
+
+ def __init__(self):
+ raise OSError(
+ "AutoVideoProcessor is designed to be instantiated "
+ "using the `AutoVideoProcessor.from_pretrained(pretrained_model_name_or_path)` method."
+ )
+
+ @classmethod
+ @replace_list_option_in_docstrings(VIDEO_PROCESSOR_MAPPING_NAMES)
+ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
+ r"""
+ Instantiate one of the video processor classes of the library from a pretrained model vocabulary.
+
+ The video processor class to instantiate is selected based on the `model_type` property of the config object
+ (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's
+ missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
+
+ List options
+
+ Params:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained video_processor hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a video processor file saved using the
+ [`~video_processing_utils.BaseVideoProcessor.save_pretrained`] method, e.g.,
+ `./my_model_directory/`.
+ - a path to a saved video processor JSON *file*, e.g.,
+ `./my_model_directory/preprocessor_config.json`.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model video processor should be cached if the
+ standard cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the video processor files and override the cached versions if
+ they exist.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+ return_unused_kwargs (`bool`, *optional*, defaults to `False`):
+ If `False`, then this function returns just the final video processor object. If `True`, then this
+ functions returns a `Tuple(video_processor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not video processor attributes: i.e., the part of
+ `kwargs` which has not been used to update `video_processor` and is otherwise ignored.
+ trust_remote_code (`bool`, *optional*, defaults to `False`):
+ Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
+ kwargs (`dict[str, Any]`, *optional*):
+ The values in kwargs of any keys which are video processor attributes will be used to override the
+ loaded values. Behavior concerning key/value pairs whose keys are *not* video processor attributes is
+ controlled by the `return_unused_kwargs` keyword parameter.
+
+
+
+ Passing `token=True` is required when you want to use a private model.
+
+
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoVideoProcessor
+
+ >>> # Download video processor from huggingface.co and cache.
+ >>> video_processor = AutoVideoProcessor.from_pretrained("llava-hf/llava-onevision-qwen2-0.5b-ov-hf")
+
+ >>> # If video processor files are in a directory (e.g. video processor was saved using *save_pretrained('./test/saved_model/')*)
+ >>> # video_processor = AutoVideoProcessor.from_pretrained("./test/saved_model/")
+ ```"""
+ config = kwargs.pop("config", None)
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
+ kwargs["_from_auto"] = True
+
+ config_dict, _ = BaseVideoProcessor.get_video_processor_dict(pretrained_model_name_or_path, **kwargs)
+ video_processor_class = config_dict.get("video_processor_type", None)
+ video_processor_auto_map = None
+ if "AutoVideoProcessor" in config_dict.get("auto_map", {}):
+ video_processor_auto_map = config_dict["auto_map"]["AutoVideoProcessor"]
+
+ # If we still don't have the video processor class, check if we're loading from a previous image processor config
+ # and if so, infer the video processor class from there.
+ if video_processor_class is None and video_processor_auto_map is None:
+ image_processor_class = config_dict.pop("image_processor_type", None)
+ if image_processor_class is not None:
+ video_processor_class_inferred = image_processor_class.replace("ImageProcessor", "VideoProcessor")
+
+ # Some models have different image processors, e.g. InternVL uses GotOCRImageProcessor
+ # We cannot use GotOCRVideoProcessor when falling back for BC and should try to infer from config later on
+ if video_processor_class_from_name(video_processor_class_inferred) is not None:
+ video_processor_class = video_processor_class_inferred
+ if "AutoImageProcessor" in config_dict.get("auto_map", {}):
+ image_processor_auto_map = config_dict["auto_map"]["AutoImageProcessor"]
+ video_processor_auto_map = image_processor_auto_map.replace("ImageProcessor", "VideoProcessor")
+
+ # If we don't find the video processor class in the video processor config, let's try the model config.
+ if video_processor_class is None and video_processor_auto_map is None:
+ if not isinstance(config, PreTrainedConfig):
+ config = AutoConfig.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+ # It could be in `config.video_processor_type``
+ video_processor_class = getattr(config, "video_processor_type", None)
+ if hasattr(config, "auto_map") and "AutoVideoProcessor" in config.auto_map:
+ video_processor_auto_map = config.auto_map["AutoVideoProcessor"]
+
+ if video_processor_class is not None:
+ video_processor_class = video_processor_class_from_name(video_processor_class)
+
+ has_remote_code = video_processor_auto_map is not None
+ has_local_code = video_processor_class is not None or type(config) in VIDEO_PROCESSOR_MAPPING
+ explicit_local_code = has_local_code and not (
+ video_processor_class or VIDEO_PROCESSOR_MAPPING[type(config)]
+ ).__module__.startswith("transformers.")
+ if has_remote_code:
+ if "--" in video_processor_auto_map:
+ upstream_repo = video_processor_auto_map.split("--")[0]
+ else:
+ upstream_repo = None
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
+ )
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ class_ref = video_processor_auto_map
+ video_processor_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
+ _ = kwargs.pop("code_revision", None)
+ video_processor_class.register_for_auto_class()
+ return video_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+ elif video_processor_class is not None:
+ return video_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+ # Last try: we use the VIDEO_PROCESSOR_MAPPING.
+ elif type(config) in VIDEO_PROCESSOR_MAPPING:
+ video_processor_class = VIDEO_PROCESSOR_MAPPING[type(config)]
+ if video_processor_class is not None:
+ return video_processor_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+ # Raise a more informative error message if torchvision isn't found, otherwise just fallback to default
+ if not is_torchvision_available():
+ raise ValueError(
+ f"{pretrained_model_name_or_path} requires `torchvision` to be installed. Please install `torchvision` and try again."
+ )
+
+ raise ValueError(
+ f"Unrecognized video processor in {pretrained_model_name_or_path}. Should have a "
+ f"`video_processor_type` key in its {VIDEO_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following "
+ f"`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in VIDEO_PROCESSOR_MAPPING_NAMES)}"
+ )
+
+ @staticmethod
+ def register(
+ config_class,
+ video_processor_class,
+ exist_ok=False,
+ ):
+ """
+ Register a new video processor for this class.
+
+ Args:
+ config_class ([`PreTrainedConfig`]):
+ The configuration corresponding to the model to register.
+ video_processor_class ([`BaseVideoProcessor`]):
+ The video processor to register.
+ """
+ VIDEO_PROCESSOR_MAPPING.register(config_class, video_processor_class, exist_ok=exist_ok)
+
+
+__all__ = ["VIDEO_PROCESSOR_MAPPING", "AutoVideoProcessor"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/distilbert/tokenization_distilbert.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/distilbert/tokenization_distilbert.py
new file mode 100644
index 0000000000000000000000000000000000000000..884c6a8d7f75a1a913e3a6cae6c8668258f54c8a
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/distilbert/tokenization_distilbert.py
@@ -0,0 +1,42 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for DistilBERT."""
+
+from ...models.bert.tokenization_bert import BertTokenizer
+
+
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
+
+
+class DistilBertTokenizer(BertTokenizer):
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(self, *args, do_lower_case: bool = True, **kwargs):
+ """
+ Construct a DistilBERT tokenizer (backed by HuggingFace's tokenizers library). Based on WordPiece.
+
+ This tokenizer inherits from [`BertTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
+
+ Args:
+ do_lower_case (`bool`, *optional*, defaults to `True`):
+ Whether or not to lowercase the input when tokenizing.
+ """
+ super().__init__(*args, do_lower_case=do_lower_case, **kwargs)
+
+
+# DistilBertTokenizerFast is an alias for DistilBertTokenizer (since BertTokenizer is already a fast tokenizer)
+DistilBertTokenizerFast = DistilBertTokenizer
+
+__all__ = ["DistilBertTokenizer", "DistilBertTokenizerFast"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanext/configuration_slanext.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanext/configuration_slanext.py
new file mode 100644
index 0000000000000000000000000000000000000000..aac603aa4a01583a9350cb82bfe7fac1ce6b2f80
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanext/configuration_slanext.py
@@ -0,0 +1,103 @@
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# This file was automatically generated from src/transformers/models/slanext/modular_slanext.py.
+# Do NOT edit this file manually as any edits will be overwritten by the generation of
+# the file from the modular. If any change should be done, please apply the change to the
+# modular_slanext.py file directly. One of our CI enforces this.
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="PaddlePaddle/SLANeXt_wired_safetensors")
+@strict
+class SLANeXtVisionConfig(PreTrainedConfig):
+ r"""
+ output_channels (`int`, *optional*, defaults to 256):
+ Dimensionality of the output channels in the Patch Encoder.
+ use_abs_pos (`bool`, *optional*, defaults to `True`):
+ Whether to use absolute position embedding.
+ use_rel_pos (`bool`, *optional*, defaults to `True`):
+ Whether to use relative position embedding.
+ window_size (`int`, *optional*, defaults to 14):
+ Window size for relative position.
+ global_attn_indexes (`list[int]`, *optional*, defaults to `[2, 5, 8, 11]`):
+ The indexes of the global attention layers.
+ mlp_dim (`int`, *optional*, defaults to 3072):
+ The dimensionality of the MLP layer in the Transformer encoder.
+ """
+
+ base_config_key = "vision_config"
+ hidden_size: int = 768
+ output_channels: int = 256
+ num_hidden_layers: int = 12
+ num_attention_heads: int = 12
+ num_channels: int = 3
+ image_size: int = 512
+ patch_size: int | list[int] | tuple[int, int] = 16
+ hidden_act: str = "gelu"
+ layer_norm_eps: float = 1e-06
+ attention_dropout: float | int = 0.0
+ initializer_range: float = 1e-10
+ qkv_bias: bool = True
+ use_abs_pos: bool = True
+ use_rel_pos: bool = True
+ window_size: int = 14
+ global_attn_indexes: list[int] | tuple[int, ...] = (2, 5, 8, 11)
+ mlp_dim: int = 3072
+
+
+@auto_docstring(checkpoint="PaddlePaddle/SLANeXt_wired_safetensors")
+@strict
+class SLANeXtConfig(PreTrainedConfig):
+ r"""
+ vision_config (`dict` or [`SLANeXtVisionConfig`], *optional*):
+ Configuration for the vision encoder. If `None`, a default [`SLANeXtVisionConfig`] is used.
+ post_conv_in_channels (`int`, *optional*, defaults to 256):
+ Number of input channels for the post-encoder convolution layer.
+ post_conv_out_channels (`int`, *optional*, defaults to 512):
+ Number of output channels for the post-encoder convolution layer.
+ out_channels (`int`, *optional*, defaults to 50):
+ Vocabulary size for the table structure token prediction head, i.e., the number of distinct structure
+ tokens the model can predict.
+ hidden_size (`int`, *optional*, defaults to 512):
+ Dimensionality of the hidden states in the attention GRU cell and the structure/location prediction heads.
+ max_text_length (`int`, *optional*, defaults to 500):
+ Maximum number of autoregressive decoding steps (tokens) for the structure and location decoder.
+ """
+
+ model_type = "slanext"
+ sub_configs = {"vision_config": SLANeXtVisionConfig}
+
+ vision_config: dict | SLANeXtVisionConfig | None = None
+ post_conv_in_channels: int = 256
+ post_conv_out_channels: int = 512
+ out_channels: int = 50
+ hidden_size: int = 512
+ max_text_length: int = 500
+
+ def __post_init__(self, **kwargs):
+ if self.vision_config is None:
+ self.vision_config = SLANeXtVisionConfig()
+ elif isinstance(self.vision_config, dict):
+ self.vision_config = SLANeXtVisionConfig(**self.vision_config)
+ super().__post_init__(**kwargs)
+
+
+__all__ = ["SLANeXtConfig"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanext/image_processing_slanext.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanext/image_processing_slanext.py
new file mode 100644
index 0000000000000000000000000000000000000000..f87aed72137f8871b1a0212bb42b6fef194454ef
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanext/image_processing_slanext.py
@@ -0,0 +1,257 @@
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# This file was automatically generated from src/transformers/models/slanext/modular_slanext.py.
+# Do NOT edit this file manually as any edits will be overwritten by the generation of
+# the file from the modular. If any change should be done, please apply the change to the
+# modular_slanext.py file directly. One of our CI enforces this.
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch
+import torchvision.transforms.v2.functional as tvF
+
+from ...image_processing_backends import TorchvisionBackend
+from ...image_processing_utils import BatchFeature
+from ...image_transforms import group_images_by_shape, reorder_images
+from ...image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, SizeDict
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import auto_docstring, is_torchdynamo_compiling, logging
+from ...utils.generic import TensorType
+from ...utils.import_utils import requires
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring
+@requires(backends=("torch",))
+class SLANeXtImageProcessor(TorchvisionBackend):
+ resample = 2 # PILImageResampling.BILINEAR
+ image_mean = IMAGENET_DEFAULT_MEAN
+ image_std = IMAGENET_DEFAULT_STD
+ size = {"height": 512, "width": 512}
+ pad_size = {"height": 512, "width": 512}
+ do_convert_rgb = True
+ do_resize = True
+ do_rescale = True
+ do_normalize = True
+ do_pad = True
+
+ def _resize(
+ self,
+ image: "torch.Tensor",
+ size: SizeDict,
+ ) -> "torch.Tensor":
+ batch_size, channels, height, width = image.shape
+ image = image.view(batch_size * channels, height, width)
+
+ device = image.device
+
+ scale = max(size.height, size.width) / max(height, width)
+ target_height = round(height * scale)
+ target_width = round(width * scale)
+
+ target_col = torch.arange(target_width, dtype=torch.float32, device=device)
+ src_col = (target_col + 0.5) * (float(width) / float(target_width)) - 0.5
+ src_col_floor = src_col.floor().to(torch.int32)
+ src_col_frac = src_col - src_col_floor.float()
+ # boundary handling
+ src_col_frac = torch.where(src_col_floor < 0, torch.zeros_like(src_col_frac), src_col_frac)
+ src_col_floor = torch.where(src_col_floor < 0, torch.zeros_like(src_col_floor), src_col_floor)
+ src_col_frac = torch.where(src_col_floor >= width - 1, torch.ones_like(src_col_frac), src_col_frac)
+ src_col_floor = torch.where(
+ src_col_floor >= width - 1, torch.full_like(src_col_floor, width - 2), src_col_floor
+ )
+ # fixed-point weights
+ weight_right = (src_col_frac * 2048 + 0.5).floor().to(torch.int32) # round-to-nearest
+ weight_left = 2048 - weight_right # (target_w,)
+ # --- row coordinate tables ---
+ target_row = torch.arange(target_height, dtype=torch.float32, device=device)
+ src_row = (target_row + 0.5) * (float(height) / float(target_height)) - 0.5
+ src_row_floor = src_row.floor().to(torch.int32)
+ src_row_frac = src_row - src_row_floor.float()
+ src_row_frac = torch.where(src_row_floor < 0, torch.zeros_like(src_row_frac), src_row_frac)
+ src_row_floor = torch.where(src_row_floor < 0, torch.zeros_like(src_row_floor), src_row_floor)
+ src_row_frac = torch.where(src_row_floor >= height - 1, torch.ones_like(src_row_frac), src_row_frac)
+ src_row_floor = torch.where(
+ src_row_floor >= height - 1, torch.full_like(src_row_floor, height - 2), src_row_floor
+ )
+ weight_bottom = (src_row_frac * 2048 + 0.5).floor().to(torch.int32)
+ weight_top = 2048 - weight_bottom # (target_h,)
+
+ image_uint8 = image.clamp(0, 255).to(torch.uint8) # (C, H, W)
+ image_int32 = image_uint8.to(torch.int32) # (C, H, W)
+ col_left = src_col_floor.long() # (target_w,)
+ col_right = (src_col_floor + 1).long() # (target_w,) safe: src_col_floor <= width-2
+ row_top = src_row_floor.long() # (target_h,)
+ row_bottom = (src_row_floor + 1).long() # (target_h,)
+ # gather 4 neighbours: (C, target_h, target_w)
+ pixel_top_left = image_int32[:, row_top[:, None], col_left[None, :]]
+ pixel_top_right = image_int32[:, row_top[:, None], col_right[None, :]]
+ pixel_bottom_left = image_int32[:, row_bottom[:, None], col_left[None, :]]
+ pixel_bottom_right = image_int32[:, row_bottom[:, None], col_right[None, :]]
+ # fixed-point bilinear: weights broadcast over (C, target_h, target_w)
+ weight_bottom_3d = weight_bottom.view(1, target_height, 1)
+ weight_top_3d = weight_top.view(1, target_height, 1)
+ weight_right_3d = weight_right.view(1, 1, target_width)
+ weight_left_3d = weight_left.view(1, 1, target_width)
+ interp = weight_top_3d * (
+ weight_left_3d * pixel_top_left + weight_right_3d * pixel_top_right
+ ) + weight_bottom_3d * (weight_left_3d * pixel_bottom_left + weight_right_3d * pixel_bottom_right)
+ interp = (interp + (1 << 21)) >> 22
+ result = interp.clamp(0, 255).to(torch.uint8) # (B*C, target_h, target_w)
+
+ return result.view(batch_size, channels, target_height, target_width).to(dtype=image.dtype)
+
+ def _preprocess(
+ self,
+ images: list["torch.Tensor"],
+ do_resize: bool,
+ size: SizeDict,
+ resample: "tvF.InterpolationMode | int | None",
+ do_center_crop: bool,
+ crop_size: SizeDict,
+ do_rescale: bool,
+ rescale_factor: float,
+ do_normalize: bool,
+ image_mean: float | list[float] | None,
+ image_std: float | list[float] | None,
+ do_pad: bool | None,
+ pad_size: SizeDict | None,
+ disable_grouping: bool | None,
+ return_tensors: str | TensorType | None,
+ **kwargs,
+ ) -> BatchFeature:
+ if resample is not None and not is_torchdynamo_compiling():
+ logger.warning_once("Resampling is not supported in SLANeXt")
+
+ # Group images by size for batched resizing
+ grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
+ resized_images_grouped = {}
+ for shape, stacked_images in grouped_images.items():
+ if do_resize:
+ stacked_images = self._resize(image=stacked_images, size=size)
+ resized_images_grouped[shape] = stacked_images
+ resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+ # Group images by size for further processing
+ # Needed in case do_resize is False, or resize returns images with different sizes
+ grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
+ processed_images_grouped = {}
+ for shape, stacked_images in grouped_images.items():
+ if do_center_crop:
+ stacked_images = self.center_crop(stacked_images, crop_size)
+ # Fused rescale and normalize
+ stacked_images = self.rescale_and_normalize(
+ stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+ )
+ processed_images_grouped[shape] = stacked_images
+ processed_images = reorder_images(processed_images_grouped, grouped_images_index)
+
+ if do_pad:
+ processed_images = self.pad(processed_images, pad_size=pad_size, disable_grouping=disable_grouping)
+
+ return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)
+
+ def __init__(self, **kwargs: Unpack[ImagesKwargs]):
+ super().__init__(**kwargs)
+ self.init_decoder()
+
+ def init_decoder(self):
+ """
+ Initialize the decoder vocabulary for table structure recognition.
+
+ Builds a character dictionary mapping HTML table structure tokens (e.g., ``, ``, `| `, colspan/
+ rowspan attributes) to integer indices. The dictionary includes special `"sos"` (start-of-sequence) and
+ `"eos"` (end-of-sequence) tokens. Merged ` | | ` tokens are used in place of standalone `` tokens
+ when applicable.
+ """
+ dict_character = [
+ "",
+ "",
+ " |
",
+ "",
+ "",
+ "
",
+ "",
+ "",
+ " | ",
+ ]
+ dict_character += [f' colspan="{i + 2}"' for i in range(19)]
+ dict_character += [f' rowspan="{i + 2}"' for i in range(19)]
+
+ if " | " not in dict_character:
+ dict_character.append(" | ")
+ if "" in dict_character:
+ dict_character.remove(" | ")
+
+ dict_character = ["sos"] + dict_character + ["eos"]
+ self.dict = {char: i for i, char in enumerate(dict_character)}
+ self.character = dict_character
+ self.td_token = [" | ", " | "]
+ self.bos_id = self.dict["sos"]
+ self.eos_id = self.dict["eos"]
+
+ def post_process_table_recognition(self, outputs):
+ """
+ Post-process the raw model outputs to decode the predicted table structure into an HTML token sequence.
+
+ Converts the model's predicted probability distributions over the structure vocabulary into a sequence of
+ HTML tokens representing the table structure. The decoded tokens are wrapped with ``, ``, and
+ `` tags to form a complete HTML table structure.
+
+ Args:
+ outputs ([`SLANeXtForTableRecognitionOutput`]):
+ Raw outputs from the SLANeXt model. The `last_hidden_state` field contains the predicted probability
+ distributions over the structure vocabulary at each decoding step, with shape
+ `(batch_size, max_text_length, num_classes)`.
+
+ Returns:
+ `dict`: A dictionary containing:
+ - **structure** (`list[str]`): The predicted HTML table structure as a list of tokens, wrapped with
+ ``, ``, and `` tags.
+ - **structure_score** (`float`): The mean confidence score across all predicted tokens.
+ """
+ self.pred = outputs.last_hidden_state
+ structure_probs = self.pred[0:1]
+ ignored_tokens = [int(self.bos_id), int(self.eos_id)]
+ end_idx = int(self.eos_id)
+
+ structure_idx = structure_probs.argmax(dim=2)
+ structure_probs = structure_probs.max(dim=2).values
+
+ structure_str_list = []
+ batch_size = structure_idx.shape[0]
+ for batch_index in range(batch_size):
+ structure_list = []
+ score_list = []
+ for position in range(structure_idx.shape[1]):
+ char_idx = int(structure_idx[batch_index, position])
+ if position > 0 and char_idx == end_idx:
+ break
+ if char_idx in ignored_tokens:
+ continue
+ text = self.character[char_idx]
+ structure_list.append(text)
+ score_list.append(structure_probs[batch_index, position])
+ structure_str_list.append(structure_list)
+ structure_score = torch.stack(score_list).mean().item()
+
+ structure = ["", "", ""] + structure_str_list[0] + ["
", "", ""]
+ return {"structure": structure, "structure_score": structure_score}
+
+
+__all__ = ["SLANeXtImageProcessor"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/vitdet/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/vitdet/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..f96b2fdf7d6247455c115ef40700507076fe6a12
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/vitdet/__init__.py
@@ -0,0 +1,27 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+ from .configuration_vitdet import *
+ from .modeling_vitdet import *
+else:
+ import sys
+
+ _file = globals()["__file__"]
+ sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_029000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_029000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b5edbba8ed43726c69159beb048b9bd045ff5a1e
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_029000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d221009eecb873d733178e1f32b36540eaec5298c006106b419d001ba3c2360
+size 515519058
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_096000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_096000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6377fab525f69624c54bccc15163dc7c3d69c9a8
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_096000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73153c0e7f00aeb1c680b93e767843b424adc5e5d89a0afe314684bc49a26e4f
+size 515519058
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_167000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_167000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cd1ff49dee7cc992042d0e729b8659b839198af
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_167000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e67f128f4b00cc4ad62f759a90617390d55ab4a62de1ba5e334e9cced469dec0
+size 515519058
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_210000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_210000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54f98503f6c4c5fc89faaf45d2406431516af048
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_stateprobadd_selfcond_ce_fast_20260531_230026/step_210000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:751cccbc439dfeeb636473d92259367833270066cfcff5038194db9989c796fc
+size 515519058