Add files using upload-large-folder tool

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260519/owt_t5_subset10k_rho0_0p25_step35k_lm1bstate_steps1024_c1024_t1p45_n8_noscore/context1024_samples.txt

@@ -0,0 +1,29 @@
+A: Res Res:sssingsingssssingssssingssingingssingingssssssingsssssinging Asss Assssingssssssingssingingingingingsasssingssingsssing sssssssssingsssssingssssssssss ingsssssingingssssingss:singsssingsssings Resssingsssssssssssingingssss Ressingsingsssingsingsingsingsssingssssss Asing Assssssingsss s singssingsssssassssingasssssssssssssss Resssingsingsssingssingssssssssingssingsingssss Reingsss ssssss sssssssssingingsssssssss thessingss Resssssss ssssssssssssingsssssingsssssss ssssssssssssssssingss Assssssssingsss Assssssssssingssingingsingssingss Assssss Aingss Assingsss Asssingssssssssssssssssssss Resassssssssssingsssssss sssingssssss Ins ssssings Ress sssingssssssssssssssss Ressing sss:ing Asssss s Resssss ssssa sssssssss s s ss Res Resssss Aingssssing ing ssss Assssssss Re Resssingssss Ressssssssssssss Resssssssssssssingsss Resssingssingssssssssss sss sssssssssings ssss s Resssssssss Re :ss ssssss Resssssss ssss Reas s : sssss Re sssssssssss Re Resss ingssssssings Re Asssss sssss Asssss Ass Re Ressssssssss Ress s Asss In ss Ressssssss Resssssssss a A ssss Ressssssssss Resingings Ressssssssssssssssssss:ss Ressssssingss Re Resssssssss Resss Resing Res ssssssssssssing Re Res Re ssss thesss Resssingss Resssssssssssssssings ssssssssssssingssss Ressssssssss Ressssssss Reingsssss Ressss Resssssssssss Res Ress s sss:sing Re
+
+---
+
+A:s Ressssss ssssssssssssssssssingssingssssssingssingssssssssssssssssssssingsssssingssssssingssingsas thesssssssssingsssssssssssssssssssssss Resss Resssssssingsingss singsssingssssssssingingsssingsssssingsss Resssssssssingsssingsss Asssssingssingingss ssssssingsssssssssing Ressingssssssssss ssssssingss Ressssssssss Resssas Re Ressssssa sssss ssssingsingssssss sssss Ressingsss Resss Inss sssssss ssssssssssssssss ssssssss:ssss sssssssssss ssssss ssss sssss Inssssssssssssssssss sssssss Resssssssssssssssssssingsss ssingsssingsssss theingss s Resssssss ssssssssssssa sss ssssssssssingsssss ssssss s Assssssssss s sssssssssssings sss sss sss Ress Resssingssss ssssss ssssings: s sssss Res:sssssssssss s s sa Resaingsingss sssss Resss Resssssssssss ss Resssss Resssssss Ressssss Ressssssingsssss :ssssss Resss Resssss ssss Resssss Ressssssss Ressssss ss Ressings s sssssssssssss sing Res Ressssingsssss Res Resssssss ings Ressss:sssssssssss thesssssssssing s s Re singssss ssssssingssss Ass Ass Re sss Reser Oversss Ress Resssingsssssss Ressss ssss Ressingssssss Re Ressssssss Ressssssssssss Re sssssssssssssss Ressss Ressings thessss Ress: ssssssssssssssserasssssingsssssssssssssss thessss Resssingsssssssssss Resssssers ssssss: Resssssingssssss Re ssssssss ss:
+
+---
+
+A Re Re:ssss Ress Reingsssssssssingssssssingingsssssssssssss Ressssssssingsassssssssssingssingsingssingsassssingssingss Reingssssingingsssssssasssasssssingssingsssssssssingssssssssssssingsss sssssss Ressssssss:ssssingsssingsssssssssingssssssssssssssssssssssssssssingsssssssssssssingsssssssssss Ressssssssssssssssssssssss ssssing as Resssssss sssssingssss Ressssssssingssssssssssssssingssssssssss ssing sssssssssssss:ssssssssssssssingsssssssingsssssingssssssssssssingssssingsasssssssssing A Resingsingssssssssss sssss Res sssssssssssss sssssssssssingssssssssssssssssing ssssingssssssssssssss Ressssssssingsssssss As ssssssssssssssssingsss Re ssssingsssssssing sing A Iningss ssss thes serssssss: Asss s ssssssss Assssssss s Asssssssassing sss:ss sssssssssss Resssssssingssss sssssss sings Rea Resss Overssingssssssssss sssss Resssssssssss ssss Resa Ressssssssingssingssssss ssingsssss Ress Overss Ress Res sings s Re sssssssssss sssss Re Resss ss Resssssss Resssssings Resssssingssss Res Resssssssing Ressssss sss Ressssssssssss sssssss Re Res Ressingssing Ressssssssssssssss Resssssssss ingsssssssssss Res ssss Res Over ssssssings sssss Re sssssss Ressingssss Re Resssssssssass: ss the Res:ingsssssssssssssssssssss thes s thesssssss Res Re Res ss Resings Resss s Ress Resss ing s:
+
+---
+
+A Re Re::ssssss Reing:ssssssssssingsssssingssssssssingsssssssssssssssssssssssingingingssssssssssingsssssings Resssersingssings sssssings Res sssssingssingssssssssssssssssssssss Resssssssingss thesss sssssssssss Iningssssing singsssssssingssss Resingsssingssingingsssssingsssss Ressssssssssingssssssss ssasssssssssssssssssssingsssssingsasingsas ss ssssserssssingssss Asssingsssssssssssssssssingssssingssingsssssssingsss Resssssssssssssingsssssssssssssssssss Insssssss Resingssssssssssssssssssss Resingss ssssssssssssssss sssssssss ingasssssssssssssssssing Resssssingsssss ss sssss ssssssingsss ssssss ssss s Aingsssssssingsssss ssss sssssings Assssingsssssssss In sssingingsssssingss Ressssssssssssingsssss As s ssssss singss Ressssssss singss sssingssssss sss s sssssssss sss ssss Res ss Res Res ssssssss Resssssss Ressssssss Ressssssssssssss Ressing ssingsssssingsssssss As s Ress sssssssssssssing Res s the ss Ins A Ressssssss Resssssssssssssss Resss Resss ssssssssssssingsssings Res Ressingssssssss ss sssssssssssingingsssss sssingssingsssssssssssss s Reing thesssss sssssssssssssssssss sssssssssssssssss Reing ofsass thessingsssssingssingssssssssssssssssssss Ressssss Res Resingsssssss thesssssssssssssssing Ress Asssssssssssssss Ressssssssssssa ssssssssss theing s:
+
+---
+
+A Re Re Re:ingingsssssssssssssssssssssssssssssssssssssssssssssssssing:singsssssings Ressssssssssssss s sssssssssingsssssssssssssingsssssssingssssingsssingsss thessssss Reingss Resssssssssingsssssssssssssssssssssingssssings Ressssss ssingsssssingssssssssssingsssss Resingsssssssssssssssssssssingsssssssssssingsssingsssssingings Resssssssingsasssssingsssssssssssssss Resssingssssings Aingsss ingssssings sssssss ssass Resss sssssssssss sssssssingingssssssingingssssssssssssssssss ingssssssssssssssssssssssssssssssingsssssssssssssss Overingsssssssss singssss Resssssssssssssssss ssssssss ssssssssingss Reingssss Resssssss Ressingssssssssss ssssss sss ssss Res Aingsssingsssssss sss Theing the sss:ssssingsssss Re sss Re sssssssssssss sssssssssssssss sssssssssss Resssssssssssss ss Res ssssssing ssss Ressssssss:ssss Ressssings ssssssssingsssssssss sssssssssssss As Res sss sssssssssssss ss Ress sssssss Res sssss Resssssssssss sss Res sssssssssasss ssingss Asssssssssssssssingsingss Res ssssssssssssssssssssssssingss sss Ressssssssssss s ssss Reingssss Re Res Re s the sssssssss:sssssss Resssss In ssssss Resss ing Ressssssersssss Re ssssssssssssing Resss Resssssssssss sssssssss ssss Resssss Resssssss Res Resssssss Ress:sssssssssssssssss
+
+---
+
+A:: Resingsssssss Asssssingsingsssssssingsingssssssssssssssssssingsingsssingsssssssssssssssssersssingsss Backingsaingsss:ssssingssssssingssssssssss Resingsasssssingsingsssssssssssssingssss sasssssingsssssingssssingssss ssssingssssssing Resssss Asssssingssssssssssssingsssingssssingsssssssingsssssss Asssssssssssssssssssssssss Assssings sssssingingssssssssssinging sssssssssingsingsssssssssssssssssssssssss ssss Resssssss ingssssssss singssssssssssings Resssssssingssssssssingsingssssssssssingss Asssss Ressssssingsingss sss Ressssss sss sssssss singingsssssingssssssss ingssssssssssssssssssssssssssss Ressssingsss ingssssssssingsss Ress Assssssssssssssssingsssss sssssssssssss Res sssss ingsssingss Asss sss sssssssssingssssss ssssss Resssssssssssss ingsss Reing sssss ss Resssssssas Reing ss s ssssssssas Re Ressssssssssss Ress Oversss ssssssssssssssd sss ssss ssssssss Resssing Ress Res Re Ressssingssssss sssss Re Oversss Ins Resssssss Assss Insssssss Re Resssssssssss Ressssssssssss s :sssssssssssssss Res ssssssingsssingss sasss Reingsssssing:sssssssss Resingsssssssssingsssss Ressssssssssss s Re ssssssssssssssssssssssss Resings sssingsssssss ss Res Resssssssss Resssssssss Ressssssssssssingssssssssssssssssss Ressss thesssingsss s Ressssssss Ressingsss Ress sssss Re
+
+---
+
+A Re::s Resssssing Ressssssingsssssssssssssssssssinging Assssingssssssssssssssssssssssssssingsingsssssings Res:ssingssssssssssingssssssssingsingssingssssssssssss sssingss Res Resssssssssssing Ressssssingsing sssssssa Ressssingssssssssssssssingasssssssssssssingssssss Assssssssssssssingssss A ssssssingssssssss ingssssssssssingss Asssingssssssssssssssssssssssss sssingssssss ssssssss Ressssss Assing ssssssssingsssingssingss Reings s sssingsssssssingsingss sssssssssssingsssingsssingssssssss:s sssssssss Resssssssssss ssingssssssss ssssssssssssssssssssss ingsssssss ssing Resssssssss Ress s ssssssssssssssingssssssssssssingssss s sssssss Res:ass ss Resssss sssssssss ssssssss sss sss ssingsssss s sssssingsss’ ssssss ss Ressssssssingssasingssasingss Insingsssssssssssssssss Resss sss Resss Asssssingsssssssssss ss ssssingsssssss Re Resingss Ressssss Ressssssss Ressss Resss Ins Ressssssssss ssss Ress Re ss s sss Resssss Assssssss sssss Ins ssss Resssss ssssss sssss sssing sssssssingss ss ingss Resssssss:singssss Ressssss Resss sssssssssingsssing Res Ress sssssssssss Reing Resssssssssss ss thes Resssss Resssssing Ressss Ressss Resssssssssssssingsssssssss Resssss Reingsss Resssssss Ress ssings Re sssssssss Res Ressssssss Reing Re Re Res Resss Ressssss Resss ssser:
+
+---
+
+A: Re:ers Ressssssssssssssings Res Ressssingingsssssingssssingsssssssssssssingingssssingsssssssssss Asingssssssssss ssssingssssssssssssssssingingsingsssssssssssssings Ressingsssssssingsssssssassssssssssssssssssssssssingsssssssaingsssss ssssssss:ssssingssssssssingsssss Reings thesssssssssss Res s:ssssssss sss Reasssssingssssingingsingssssssssssssssss Re ss Ass Ressssssingssssssss sssingsssssingsss Res Asssssssssssssssssssss sssss ssssss sssssingsss Assssingssssssss Ressss Asingsssingssssssssssssssingsingssssssssssssss ssings Resssssssss ssssssssing Insssssssssssssssssssssssing Resss Ressss sssssssssssssssss s Asssss Assssasssingsssssss sssssssssssssingingsing:sssssss As ssingsss sssssssss ingssssingings Ressssssssss sssssssssingssssssssssing s ssssssssssss ssssssss sss s ssssssssssssssss ssss ssssssssss s sss Ress Resss Ressssssssssss Resss sssss:ss ss:sssss the Resssssssss sssssssss Res Resss Resss ssssss Resssss As Assssssssasssing Resss Re Ressssssssssing Resssss thesssssssssingsssssssss ssss Insssssssssssssssss Resssssssssing Ress Resssingsssssssssssingsssingsssass Resssss thesssss Ressss Resssssss Ressssssings Resa Re the Resssss sss Resssssssssssssssssssssss Resssingss thessssasssss sss sssssingssssss Resssssssssssssingsssssssss:s

LTA_openwebtext_dualt/logs/qwen_service/qwen3p5_35b_a3b_transformers_gpu2_3_20260608_200311.log

@@ -0,0 +1,67 @@
+[transformers] `torch_dtype` is deprecated! Use `dtype` instead!
+[transformers] The fast path is not available because one of the required library is not installed. Falling back to torch implementation. To install follow https://github.com/fla-org/flash-linear-attention#installation and https://github.com/Dao-AILab/causal-conv1d
+
+INFO:     Started server process [817067]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
+INFO:     127.0.0.1:39348 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:38936 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:42672 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     127.0.0.1:53660 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:45760 - "GET / HTTP/1.1" 404 Not Found
+INFO:     127.0.0.1:37066 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:60836 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:58018 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:44326 - "GET / HTTP/1.1" 404 Not Found
+INFO:     193.205.210.75:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     127.0.0.1:40530 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:60692 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO:     127.0.0.1:49442 - "GET / HTTP/1.1" 404 Not Found
+INFO:     127.0.0.1:58800 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     127.0.0.1:47506 - "GET / HTTP/1.1" 404 Not Found
+INFO:     127.0.0.1:54578 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     127.0.0.1:34000 - "GET / HTTP/1.1" 404 Not Found
+INFO:     77.97.20.28:0 - "POST /chat/completions HTTP/1.1" 404 Not Found
+INFO:     77.97.20.28:0 - "POST /chat/completions HTTP/1.1" 404 Not Found
+INFO:     77.97.20.28:0 - "POST /chat/completions HTTP/1.1" 404 Not Found
+INFO:     77.97.20.28:0 - "POST /chat/completions HTTP/1.1" 404 Not Found
+INFO:     77.97.20.28:0 - "POST /chat/completions HTTP/1.1" 404 Not Found
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 401 Unauthorized
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 401 Unauthorized
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 401 Unauthorized
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 401 Unauthorized
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 401 Unauthorized
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "GET /healthz HTTP/1.1" 200 OK
+INFO:     193.205.210.75:0 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+INFO:     77.97.20.28:0 - "POST /v1/chat/completions HTTP/1.1" 401 Unauthorized
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [817067]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/_usd_builtins.py

@@ -0,0 +1,112 @@
+"""
+    pygments.lexers._usd_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    A collection of known USD-related keywords, attributes, and types.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+COMMON_ATTRIBUTES = [
+    "extent",
+    "xformOpOrder",
+]
+
+KEYWORDS = [
+    "class",
+    "clips",
+    "custom",
+    "customData",
+    "def",
+    "dictionary",
+    "inherits",
+    "over",
+    "payload",
+    "references",
+    "rel",
+    "subLayers",
+    "timeSamples",
+    "uniform",
+    "variantSet",
+    "variantSets",
+    "variants",
+]
+
+OPERATORS = [
+    "add",
+    "append",
+    "delete",
+    "prepend",
+    "reorder",
+]
+
+SPECIAL_NAMES = [
+    "active",
+    "apiSchemas",
+    "defaultPrim",
+    "elementSize",
+    "endTimeCode",
+    "hidden",
+    "instanceable",
+    "interpolation",
+    "kind",
+    "startTimeCode",
+    "upAxis",
+]
+
+TYPES = [
+    "asset",
+    "bool",
+    "color3d",
+    "color3f",
+    "color3h",
+    "color4d",
+    "color4f",
+    "color4h",
+    "double",
+    "double2",
+    "double3",
+    "double4",
+    "float",
+    "float2",
+    "float3",
+    "float4",
+    "frame4d",
+    "half",
+    "half2",
+    "half3",
+    "half4",
+    "int",
+    "int2",
+    "int3",
+    "int4",
+    "keyword",
+    "matrix2d",
+    "matrix3d",
+    "matrix4d",
+    "normal3d",
+    "normal3f",
+    "normal3h",
+    "point3d",
+    "point3f",
+    "point3h",
+    "quatd",
+    "quatf",
+    "quath",
+    "string",
+    "syn",
+    "token",
+    "uchar",
+    "uchar2",
+    "uchar3",
+    "uchar4",
+    "uint",
+    "uint2",
+    "uint3",
+    "uint4",
+    "usdaType",
+    "vector3d",
+    "vector3f",
+    "vector3h",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/ecl.py

@@ -0,0 +1,144 @@
+"""
+    pygments.lexers.ecl
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the ECL language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, words
+from pygments.token import Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['ECLLexer']
+
+
+class ECLLexer(RegexLexer):
+    """
+    Lexer for the declarative big-data ECL language.
+    """
+
+    name = 'ECL'
+    url = 'https://hpccsystems.com/training/documentation/ecl-language-reference/html'
+    aliases = ['ecl']
+    filenames = ['*.ecl']
+    mimetypes = ['application/x-ecl']
+    version_added = '1.5'
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('statements'),
+        ],
+        'whitespace': [
+            (r'\s+', Whitespace),
+            (r'\/\/.*', Comment.Single),
+            (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
+        ],
+        'statements': [
+            include('types'),
+            include('keywords'),
+            include('functions'),
+            include('hash'),
+            (r'"', String, 'string'),
+            (r'\'', String, 'string'),
+            (r'(\d+\.\d*|\.\d+|\d+)e[+-]?\d+[lu]*', Number.Float),
+            (r'(\d+\.\d*|\.\d+|\d+f)f?', Number.Float),
+            (r'0x[0-9a-f]+[lu]*', Number.Hex),
+            (r'0[0-7]+[lu]*', Number.Oct),
+            (r'\d+[lu]*', Number.Integer),
+            (r'[~!%^&*+=|?:<>/-]+', Operator),
+            (r'[{}()\[\],.;]', Punctuation),
+            (r'[a-z_]\w*', Name),
+        ],
+        'hash': [
+            (r'^#.*$', Comment.Preproc),
+        ],
+        'types': [
+            (r'(RECORD|END)\D', Keyword.Declaration),
+            (r'((?:ASCII|BIG_ENDIAN|BOOLEAN|DATA|DECIMAL|EBCDIC|INTEGER|PATTERN|'
+             r'QSTRING|REAL|RECORD|RULE|SET OF|STRING|TOKEN|UDECIMAL|UNICODE|'
+             r'UNSIGNED|VARSTRING|VARUNICODE)\d*)(\s+)',
+             bygroups(Keyword.Type, Whitespace)),
+        ],
+        'keywords': [
+            (words((
+                'APPLY', 'ASSERT', 'BUILD', 'BUILDINDEX', 'EVALUATE', 'FAIL',
+                'KEYDIFF', 'KEYPATCH', 'LOADXML', 'NOTHOR', 'NOTIFY', 'OUTPUT',
+                'PARALLEL', 'SEQUENTIAL', 'SOAPCALL', 'CHECKPOINT', 'DEPRECATED',
+                'FAILCODE', 'FAILMESSAGE', 'FAILURE', 'GLOBAL', 'INDEPENDENT',
+                'ONWARNING', 'PERSIST', 'PRIORITY', 'RECOVERY', 'STORED', 'SUCCESS',
+                'WAIT', 'WHEN'), suffix=r'\b'),
+             Keyword.Reserved),
+            # These are classed differently, check later
+            (words((
+                'ALL', 'AND', 'ANY', 'AS', 'ATMOST', 'BEFORE', 'BEGINC++', 'BEST',
+                'BETWEEN', 'CASE', 'CONST', 'COUNTER', 'CSV', 'DESCEND', 'ENCRYPT',
+                'ENDC++', 'ENDMACRO', 'EXCEPT', 'EXCLUSIVE', 'EXPIRE', 'EXPORT',
+                'EXTEND', 'FALSE', 'FEW', 'FIRST', 'FLAT', 'FULL', 'FUNCTION',
+                'GROUP', 'HEADER', 'HEADING', 'HOLE', 'IFBLOCK', 'IMPORT', 'IN',
+                'JOINED', 'KEEP', 'KEYED', 'LAST', 'LEFT', 'LIMIT', 'LOAD', 'LOCAL',
+                'LOCALE', 'LOOKUP', 'MACRO', 'MANY', 'MAXCOUNT', 'MAXLENGTH',
+                'MIN SKEW', 'MODULE', 'INTERFACE', 'NAMED', 'NOCASE', 'NOROOT',
+                'NOSCAN', 'NOSORT', 'NOT', 'OF', 'ONLY', 'OPT', 'OR', 'OUTER',
+                'OVERWRITE', 'PACKED', 'PARTITION', 'PENALTY', 'PHYSICALLENGTH',
+                'PIPE', 'QUOTE', 'RELATIONSHIP', 'REPEAT', 'RETURN', 'RIGHT',
+                'SCAN', 'SELF', 'SEPARATOR', 'SERVICE', 'SHARED', 'SKEW', 'SKIP',
+                'SQL', 'STORE', 'TERMINATOR', 'THOR', 'THRESHOLD', 'TOKEN',
+                'TRANSFORM', 'TRIM', 'TRUE', 'TYPE', 'UNICODEORDER', 'UNSORTED',
+                'VALIDATE', 'VIRTUAL', 'WHOLE', 'WILD', 'WITHIN', 'XML', 'XPATH',
+                '__COMPRESSED__'), suffix=r'\b'),
+             Keyword.Reserved),
+        ],
+        'functions': [
+            (words((
+                'ABS', 'ACOS', 'ALLNODES', 'ASCII', 'ASIN', 'ASSTRING', 'ATAN',
+                'ATAN2', 'AVE', 'CASE', 'CHOOSE', 'CHOOSEN', 'CHOOSESETS',
+                'CLUSTERSIZE', 'COMBINE', 'CORRELATION', 'COS', 'COSH', 'COUNT',
+                'COVARIANCE', 'CRON', 'DATASET', 'DEDUP', 'DEFINE', 'DENORMALIZE',
+                'DISTRIBUTE', 'DISTRIBUTED', 'DISTRIBUTION', 'EBCDIC', 'ENTH',
+                'ERROR', 'EVALUATE', 'EVENT', 'EVENTEXTRA', 'EVENTNAME', 'EXISTS',
+                'EXP', 'FAILCODE', 'FAILMESSAGE', 'FETCH', 'FROMUNICODE',
+                'GETISVALID', 'GLOBAL', 'GRAPH', 'GROUP', 'HASH', 'HASH32',
+                'HASH64', 'HASHCRC', 'HASHMD5', 'HAVING', 'IF', 'INDEX',
+                'INTFORMAT', 'ISVALID', 'ITERATE', 'JOIN', 'KEYUNICODE', 'LENGTH',
+                'LIBRARY', 'LIMIT', 'LN', 'LOCAL', 'LOG', 'LOOP', 'MAP', 'MATCHED',
+                'MATCHLENGTH', 'MATCHPOSITION', 'MATCHTEXT', 'MATCHUNICODE', 'MAX',
+                'MERGE', 'MERGEJOIN', 'MIN', 'NOLOCAL', 'NONEMPTY', 'NORMALIZE',
+                'PARSE', 'PIPE', 'POWER', 'PRELOAD', 'PROCESS', 'PROJECT', 'PULL',
+                'RANDOM', 'RANGE', 'RANK', 'RANKED', 'REALFORMAT', 'RECORDOF',
+                'REGEXFIND', 'REGEXREPLACE', 'REGROUP', 'REJECTED', 'ROLLUP',
+                'ROUND', 'ROUNDUP', 'ROW', 'ROWDIFF', 'SAMPLE', 'SET', 'SIN',
+                'SINH', 'SIZEOF', 'SOAPCALL', 'SORT', 'SORTED', 'SQRT', 'STEPPED',
+                'STORED', 'SUM', 'TABLE', 'TAN', 'TANH', 'THISNODE', 'TOPN',
+                'TOUNICODE', 'TRANSFER', 'TRIM', 'TRUNCATE', 'TYPEOF', 'UNGROUP',
+                'UNICODEORDER', 'VARIANCE', 'WHICH', 'WORKUNIT', 'XMLDECODE',
+                'XMLENCODE', 'XMLTEXT', 'XMLUNICODE'), suffix=r'\b'),
+             Name.Function),
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\'', String, '#pop'),
+            (r'[^"\']+', String),
+        ],
+    }
+
+    def analyse_text(text):
+        """This is very difficult to guess relative to other business languages.
+        -> in conjunction with BEGIN/END seems relatively rare though."""
+        result = 0
+
+        if '->' in text:
+            result += 0.01
+        if 'BEGIN' in text:
+            result += 0.01
+        if 'END' in text:
+            result += 0.01
+
+        return result

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/gcodelexer.py

@@ -0,0 +1,35 @@
+"""
+    pygments.lexers.gcodelexer
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the G Code Language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Comment, Name, Text, Keyword, Number
+
+__all__ = ['GcodeLexer']
+
+
+class GcodeLexer(RegexLexer):
+    """
+    For gcode source code.
+    """
+    name = 'g-code'
+    aliases = ['gcode']
+    filenames = ['*.gcode']
+    url = 'https://en.wikipedia.org/wiki/G-code'
+    version_added = '2.9'
+
+    tokens = {
+        'root': [
+            (r';.*\n', Comment),
+            (r'^[gmGM]\d{1,4}\s', Name.Builtin),  # M or G commands
+            (r'([^gGmM])([+-]?\d*[.]?\d+)', bygroups(Keyword, Number)),
+            (r'\s', Text.Whitespace),
+            (r'.*\n', Text),
+        ]
+    }

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/hexdump.py

@@ -0,0 +1,102 @@
+"""
+    pygments.lexers.hexdump
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for hexadecimal dumps.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import Name, Number, String, Punctuation, Whitespace
+
+__all__ = ['HexdumpLexer']
+
+
+class HexdumpLexer(RegexLexer):
+    """
+    For typical hex dump output formats by the UNIX and GNU/Linux tools ``hexdump``,
+    ``hd``, ``hexcat``, ``od`` and ``xxd``, and the DOS tool ``DEBUG``. For example:
+
+    .. sourcecode:: hexdump
+
+        00000000  7f 45 4c 46 02 01 01 00  00 00 00 00 00 00 00 00  |.ELF............|
+        00000010  02 00 3e 00 01 00 00 00  c5 48 40 00 00 00 00 00  |..>......H@.....|
+
+    The specific supported formats are the outputs of:
+
+    * ``hexdump FILE``
+    * ``hexdump -C FILE`` -- the `canonical` format used in the example.
+    * ``hd FILE`` -- same as ``hexdump -C FILE``.
+    * ``hexcat FILE``
+    * ``od -t x1z FILE``
+    * ``xxd FILE``
+    * ``DEBUG.EXE FILE.COM`` and entering ``d`` to the prompt.
+    """
+    name = 'Hexdump'
+    aliases = ['hexdump']
+    url = 'https://en.wikipedia.org/wiki/Hex_dump'
+    version_added = '2.1'
+
+    hd = r'[0-9A-Ha-h]'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            include('offset'),
+            (r'('+hd+r'{2})(\-)('+hd+r'{2})',
+             bygroups(Number.Hex, Punctuation, Number.Hex)),
+            (hd+r'{2}', Number.Hex),
+            (r'(\s{2,3})(\>)(.{16})(\<)$',
+             bygroups(Whitespace, Punctuation, String, Punctuation), 'bracket-strings'),
+            (r'(\s{2,3})(\|)(.{16})(\|)$',
+             bygroups(Whitespace, Punctuation, String, Punctuation), 'piped-strings'),
+            (r'(\s{2,3})(\>)(.{1,15})(\<)$',
+             bygroups(Whitespace, Punctuation, String, Punctuation)),
+            (r'(\s{2,3})(\|)(.{1,15})(\|)$',
+             bygroups(Whitespace, Punctuation, String, Punctuation)),
+            (r'(\s{2,3})(.{1,15})$', bygroups(Whitespace, String)),
+            (r'(\s{2,3})(.{16}|.{20})$', bygroups(Whitespace, String), 'nonpiped-strings'),
+            (r'\s', Whitespace),
+            (r'^\*', Punctuation),
+        ],
+        'offset': [
+            (r'^('+hd+'+)(:)', bygroups(Name.Label, Punctuation), 'offset-mode'),
+            (r'^'+hd+'+', Name.Label),
+        ],
+        'offset-mode': [
+            (r'\s', Whitespace, '#pop'),
+            (hd+'+', Name.Label),
+            (r':', Punctuation)
+        ],
+        'piped-strings': [
+            (r'\n', Whitespace),
+            include('offset'),
+            (hd+r'{2}', Number.Hex),
+            (r'(\s{2,3})(\|)(.{1,16})(\|)$',
+             bygroups(Whitespace, Punctuation, String, Punctuation)),
+            (r'\s', Whitespace),
+            (r'^\*', Punctuation),
+        ],
+        'bracket-strings': [
+            (r'\n', Whitespace),
+            include('offset'),
+            (hd+r'{2}', Number.Hex),
+            (r'(\s{2,3})(\>)(.{1,16})(\<)$',
+             bygroups(Whitespace, Punctuation, String, Punctuation)),
+            (r'\s', Whitespace),
+            (r'^\*', Punctuation),
+        ],
+        'nonpiped-strings': [
+            (r'\n', Whitespace),
+            include('offset'),
+            (r'('+hd+r'{2})(\-)('+hd+r'{2})',
+             bygroups(Number.Hex, Punctuation, Number.Hex)),
+            (hd+r'{2}', Number.Hex),
+            (r'(\s{19,})(.{1,20}?)$', bygroups(Whitespace, String)),
+            (r'(\s{2,3})(.{1,20})$', bygroups(Whitespace, String)),
+            (r'\s', Whitespace),
+            (r'^\*', Punctuation),
+        ],
+    }

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/ldap.py

@@ -0,0 +1,155 @@
+"""
+    pygments.lexers.ldap
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Pygments lexers for LDAP.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+
+from pygments.lexer import RegexLexer, bygroups, default
+from pygments.token import Operator, Comment, Keyword, Literal, Name, String, \
+    Number, Punctuation, Whitespace, Escape
+
+__all__ = ['LdifLexer', 'LdaprcLexer']
+
+
+class LdifLexer(RegexLexer):
+
+    """
+    Lexer for LDIF
+    """
+
+    name = 'LDIF'
+    aliases = ['ldif']
+    filenames = ['*.ldif']
+    mimetypes = ["text/x-ldif"]
+    url = "https://datatracker.ietf.org/doc/html/rfc2849"
+    version_added = '2.17'
+
+    tokens = {
+        'root': [
+            (r'\s*\n', Whitespace),
+            (r'(-)(\n)', bygroups(Punctuation, Whitespace)),
+            (r'(#.*)(\n)', bygroups(Comment.Single, Whitespace)),
+            (r'(version)(:)([ \t]*)(.*)([ \t]*\n)', bygroups(Keyword,
+             Punctuation, Whitespace, Number.Integer, Whitespace)),
+            (r'(control)(:)([ \t]*)([\.0-9]+)([ \t]+)((?:true|false)?)([ \t]*)',
+                bygroups(Keyword, Punctuation, Whitespace, Name.Other, Whitespace, Keyword, Whitespace), "after-control"),
+            (r'(deleteoldrdn)(:)([ \n]*)([0-1]+)([ \t]*\n)',
+             bygroups(Keyword, Punctuation, Whitespace, Number, Whitespace)),
+            (r'(add|delete|replace)(::?)(\s*)(.*)([ \t]*\n)', bygroups(
+                Keyword, Punctuation, Whitespace, Name.Attribute, Whitespace)),
+            (r'(changetype)(:)([ \t]*)([a-z]*)([ \t]*\n)',
+             bygroups(Keyword, Punctuation, Whitespace, Keyword, Whitespace)),
+            (r'(dn|newrdn)(::)', bygroups(Keyword, Punctuation), "base64-dn"),
+            (r'(dn|newrdn)(:)', bygroups(Keyword, Punctuation), "dn"),
+            (r'(objectclass)(:)([ \t]*)([^ \t\n]*)([ \t]*\n)', bygroups(
+                Keyword, Punctuation, Whitespace, Name.Class, Whitespace)),
+            (r'([a-zA-Z]*|[0-9][0-9\.]*[0-9])(;)',
+             bygroups(Name.Attribute, Punctuation), "property"),
+            (r'([a-zA-Z]*|[0-9][0-9\.]*[0-9])(:<)',
+             bygroups(Name.Attribute, Punctuation), "url"),
+            (r'([a-zA-Z]*|[0-9][0-9\.]*[0-9])(::?)',
+             bygroups(Name.Attribute, Punctuation), "value"),
+        ],
+        "after-control": [
+            (r":<", Punctuation, ("#pop", "url")),
+            (r"::?", Punctuation, ("#pop", "value")),
+            default("#pop"),
+        ],
+        'property': [
+            (r'([-a-zA-Z0-9]*)(;)', bygroups(Name.Property, Punctuation)),
+            (r'([-a-zA-Z0-9]*)(:<)',
+             bygroups(Name.Property, Punctuation), ("#pop", "url")),
+            (r'([-a-zA-Z0-9]*)(::?)',
+             bygroups(Name.Property, Punctuation), ("#pop", "value")),
+        ],
+        'value': [
+            (r'(\s*)([^\n]+\S)(\n )',
+             bygroups(Whitespace, String, Whitespace)),
+            (r'(\s*)([^\n]+\S)(\n)',
+             bygroups(Whitespace, String, Whitespace), "#pop"),
+        ],
+        'url': [
+            (r'([ \t]*)(\S*)([ \t]*\n )',
+             bygroups(Whitespace, Comment.PreprocFile, Whitespace)),
+            (r'([ \t]*)(\S*)([ \t]*\n)', bygroups(Whitespace,
+             Comment.PreprocFile, Whitespace), "#pop"),
+        ],
+        "dn": [
+            (r'([ \t]*)([-a-zA-Z0-9\.]+)(=)', bygroups(Whitespace,
+             Name.Attribute, Operator), ("#pop", "dn-value")),
+        ],
+        "dn-value": [
+            (r'\\[^\n]', Escape),
+            (r',', Punctuation, ("#pop", "dn")),
+            (r'\+', Operator, ("#pop", "dn")),
+            (r'[^,\+\n]+', String),
+            (r'\n ', Whitespace),
+            (r'\n', Whitespace, "#pop"),
+        ],
+        "base64-dn": [
+            (r'([ \t]*)([^ \t\n][^ \t\n]*[^\n])([ \t]*\n )',
+             bygroups(Whitespace, Name, Whitespace)),
+            (r'([ \t]*)([^ \t\n][^ \t\n]*[^\n])([ \t]*\n)',
+             bygroups(Whitespace, Name, Whitespace), "#pop"),
+        ]
+    }
+
+
+class LdaprcLexer(RegexLexer):
+    """
+    Lexer for OpenLDAP configuration files.
+    """
+
+    name = 'LDAP configuration file'
+    aliases = ['ldapconf', 'ldaprc']
+    filenames = ['.ldaprc', 'ldaprc', 'ldap.conf']
+    mimetypes = ["text/x-ldapconf"]
+    url = 'https://www.openldap.org/software//man.cgi?query=ldap.conf&sektion=5&apropos=0&manpath=OpenLDAP+2.4-Release'
+    version_added = '2.17'
+
+    _sasl_keywords = r'SASL_(?:MECH|REALM|AUTHCID|AUTHZID|CBINDING)'
+    _tls_keywords = r'TLS_(?:CACERT|CACERTDIR|CERT|ECNAME|KEY|CIPHER_SUITE|PROTOCOL_MIN|RANDFILE|CRLFILE)'
+    _literal_keywords = rf'(?:URI|SOCKET_BIND_ADDRESSES|{_sasl_keywords}|{_tls_keywords})'
+    _boolean_keywords = r'GSSAPI_(?:ALLOW_REMOTE_PRINCIPAL|ENCRYPT|SIGN)|REFERRALS|SASL_NOCANON'
+    _integer_keywords = r'KEEPALIVE_(?:IDLE|PROBES|INTERVAL)|NETWORK_TIMEOUT|PORT|SIZELIMIT|TIMELIMIT|TIMEOUT'
+    _secprops = r'none|noanonymous|noplain|noactive|nodict|forwardsec|passcred|(?:minssf|maxssf|maxbufsize)=\d+'
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r'#.*', Comment.Single),
+            (r'\s+', Whitespace),
+            (rf'({_boolean_keywords})(\s+)(on|true|yes|off|false|no)$',
+             bygroups(Keyword, Whitespace, Keyword.Constant)),
+            (rf'({_integer_keywords})(\s+)(\d+)',
+             bygroups(Keyword, Whitespace, Number.Integer)),
+            (r'(VERSION)(\s+)(2|3)', bygroups(Keyword, Whitespace, Number.Integer)),
+            # Constants
+            (r'(DEREF)(\s+)(never|searching|finding|always)',
+             bygroups(Keyword, Whitespace, Keyword.Constant)),
+            (rf'(SASL_SECPROPS)(\s+)((?:{_secprops})(?:,{_secprops})*)',
+             bygroups(Keyword, Whitespace, Keyword.Constant)),
+            (r'(SASL_CBINDING)(\s+)(none|tls-unique|tls-endpoint)',
+             bygroups(Keyword, Whitespace, Keyword.Constant)),
+            (r'(TLS_REQ(?:CERT|SAN))(\s+)(allow|demand|hard|never|try)',
+             bygroups(Keyword, Whitespace, Keyword.Constant)),
+            (r'(TLS_CRLCHECK)(\s+)(none|peer|all)',
+             bygroups(Keyword, Whitespace, Keyword.Constant)),
+            # Literals
+            (r'(BASE|BINDDN)(\s+)(\S+)$',
+             bygroups(Keyword, Whitespace, Literal)),
+            # Accepts hostname with or without port.
+            (r'(HOST)(\s+)([a-z0-9]+)((?::(\d+))?)',
+             bygroups(Keyword, Whitespace, Literal, Number.Integer)),
+            (rf'({_literal_keywords})(\s+)(\S+)$',
+             bygroups(Keyword, Whitespace, Literal)),
+        ],
+    }

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/mips.py

@@ -0,0 +1,130 @@
+"""
+    pygments.lexers.mips
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for MIPS assembly.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Whitespace, Comment, String, Keyword, Name, Text
+
+__all__ = ["MIPSLexer"]
+
+
+class MIPSLexer(RegexLexer):
+    """
+    A MIPS Assembly Lexer.
+
+    Based on the Emacs major mode by hlissner:
+    https://github.com/hlissner/emacs-mips-mode
+    """
+
+    name = 'MIPS'
+    aliases = ['mips']
+    version_added = ''
+    # TODO: add '*.s' and '*.asm', which will require designing an analyse_text
+    # method for this lexer and refactoring those from Gas and Nasm in order to
+    # have relatively reliable detection
+    filenames = ['*.mips', '*.MIPS']
+    url = 'https://mips.com'
+
+    keywords = [
+        # Arithmetic insturctions
+        "add", "sub", "subu", "addi", "subi", "addu", "addiu",
+        # Multiplication/division
+        "mul", "mult", "multu", "mulu", "madd", "maddu", "msub", "msubu", "div", "divu",
+        # Bitwise operations
+        "and", "or", "nor", "xor", "andi", "ori", "xori", "clo", "clz",
+        # Shifts
+        "sll", "srl", "sllv", "srlv", "sra", "srav",
+        # Comparisons
+        "slt", "sltu", "slti", "sltiu",
+        # Move data
+        "mfhi", "mthi", "mflo", "mtlo", "movn", "movz", "movf", "movt",
+        # Jump
+        "j", "jal", "jalr", "jr",
+        # branch
+        "bc1f", "bc1t", "beq", "bgez", "bgezal", "bgtz", "blez", "bltzal", "bltz", "bne",
+        # Load
+        "lui", "lb", "lbu", "lh", "lhu", "lw", "lwcl", "lwl", "lwr",
+        # Store
+        "sb", "sh", "sw", "swl", "swr", # coproc: swc1 sdc1
+        # Concurrent load/store
+        "ll", "sc",
+        # Trap handling
+        "teq", "teqi", "tne", "tneqi", "tge", "tgeu", "tgei", "tgeiu", "tlt", "tltu", "tlti",
+        "tltiu",
+        # Exception / Interrupt
+        "eret", "break", "bop", "syscall",
+        # --- Floats -----------------------------------------------------
+        # Arithmetic
+        "add.s", "add.d", "sub.s", "sub.d", "mul.s", "mul.d", "div.s", "div.d", "neg.d",
+        "neg.s",
+        # Comparison
+        "c.e.d", "c.e.s", "c.le.d", "c.le.s", "c.lt.s", "c.lt.d", # "c.gt.s", "c.gt.d",
+        "madd.s", "madd.d", "msub.s", "msub.d",
+        # Move Floats
+        "mov.d", "move.s", "movf.d", "movf.s", "movt.d", "movt.s", "movn.d", "movn.s",
+        "movnzd", "movz.s", "movz.d",
+        # Conversion
+        "cvt.d.s", "cvt.d.w", "cvt.s.d", "cvt.s.w", "cvt.w.d", "cvt.w.s", "trunc.w.d",
+        "trunc.w.s",
+        # Math
+        "abs.s", "abs.d", "sqrt.s", "sqrt.d", "ceil.w.d", "ceil.w.s", "floor.w.d",
+        "floor.w.s", "round.w.d", "round.w.s",
+    ]
+
+    pseudoinstructions = [
+        # Arithmetic & logical
+        "rem", "remu", "mulo", "mulou", "abs", "neg", "negu", "not", "rol", "ror",
+        # branches
+        "b", "beqz", "bge", "bgeu", "bgt", "bgtu", "ble", "bleu", "blt", "bltu", "bnez",
+        # loads
+        "la", "li", "ld", "ulh", "ulhu", "ulw",
+        # Store
+        "sd", "ush", "usw",
+        # move
+        "move", # coproc: "mfc1.d",
+        # comparisons
+        "sgt", "sgtu", "sge", "sgeu", "sle", "sleu", "sne", "seq",
+        # --- Floats -----------------------------------------------------
+        # load-store
+        "l.d", "l.s", "s.d", "s.s",
+    ]
+
+    directives = [
+        ".align", ".ascii", ".asciiz", ".byte", ".data", ".double", ".extern", ".float",
+        ".globl", ".half", ".kdata", ".ktext", ".space", ".text", ".word",
+    ]
+
+    deprecated = [
+        "beql", "bnel", "bgtzl", "bgezl", "bltzl", "blezl", "bltzall", "bgezall",
+    ]
+
+    tokens = {
+        'root': [
+            (r'\s+', Whitespace),
+            (r'#.*', Comment),
+            (r'"', String, 'string'),
+            (r'-?[0-9]+?', Keyword.Constant),
+            (r'\w*:', Name.Function),
+            (words(deprecated, suffix=r'\b'), Keyword.Pseudo), # need warning face
+            (words(pseudoinstructions, suffix=r'\b'), Name.Variable),
+            (words(keywords, suffix=r'\b'), Keyword),
+            (r'[slm][ftwd]c[0-9]([.]d)?', Keyword),
+            (r'\$(f?[0-2][0-9]|f?3[01]|[ft]?[0-9]|[vk][01]|a[0-3]|s[0-7]|[gsf]p|ra|at|zero)',
+             Keyword.Type),
+            (words(directives, suffix=r'\b'), Name.Entity), # Preprocessor?
+            (r':|,|;|\{|\}|=>|@|\$|=', Name.Builtin),
+            (r'\w+', Text),
+            (r'.', Text),
+        ],
+        'string': [
+            (r'\\.', String.Escape),
+            (r'"', String, '#pop'),
+            (r'[^\\"]+', String),
+        ],
+    }

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/other.py

@@ -0,0 +1,41 @@
+"""
+    pygments.lexers.other
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Just export lexer classes previously contained in this module.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# ruff: noqa: F401
+from pygments.lexers.sql import SqlLexer, MySqlLexer, SqliteConsoleLexer
+from pygments.lexers.shell import BashLexer, BashSessionLexer, BatchLexer, \
+    TcshLexer
+from pygments.lexers.robotframework import RobotFrameworkLexer
+from pygments.lexers.testing import GherkinLexer
+from pygments.lexers.esoteric import BrainfuckLexer, BefungeLexer, RedcodeLexer
+from pygments.lexers.prolog import LogtalkLexer
+from pygments.lexers.snobol import SnobolLexer
+from pygments.lexers.rebol import RebolLexer
+from pygments.lexers.configs import KconfigLexer, Cfengine3Lexer
+from pygments.lexers.modeling import ModelicaLexer
+from pygments.lexers.scripting import AppleScriptLexer, MOOCodeLexer, \
+    HybrisLexer
+from pygments.lexers.graphics import PostScriptLexer, GnuplotLexer, \
+    AsymptoteLexer, PovrayLexer
+from pygments.lexers.business import ABAPLexer, OpenEdgeLexer, \
+    GoodDataCLLexer, MaqlLexer
+from pygments.lexers.automation import AutoItLexer, AutohotkeyLexer
+from pygments.lexers.dsls import ProtoBufLexer, BroLexer, PuppetLexer, \
+    MscgenLexer, VGLLexer
+from pygments.lexers.basic import CbmBasicV2Lexer
+from pygments.lexers.pawn import SourcePawnLexer, PawnLexer
+from pygments.lexers.ecl import ECLLexer
+from pygments.lexers.urbi import UrbiscriptLexer
+from pygments.lexers.smalltalk import SmalltalkLexer, NewspeakLexer
+from pygments.lexers.installers import NSISLexer, RPMSpecLexer
+from pygments.lexers.textedit import AwkLexer
+from pygments.lexers.smv import NuSMVLexer
+
+__all__ = []

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/qlik.py

@@ -0,0 +1,117 @@
+"""
+    pygments.lexers.qlik
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the qlik scripting language
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+from pygments.lexers._qlik_builtins import OPERATORS_LIST, STATEMENT_LIST, \
+    SCRIPT_FUNCTIONS, CONSTANT_LIST
+
+__all__ = ["QlikLexer"]
+
+
+class QlikLexer(RegexLexer):
+    """
+    Lexer for qlik code, including .qvs files
+    """
+
+    name = "Qlik"
+    aliases = ["qlik", "qlikview", "qliksense", "qlikscript"]
+    filenames = ["*.qvs", "*.qvw"]
+    url = "https://qlik.com"
+    version_added = '2.12'
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        # Handle multi-line comments
+        "comment": [
+            (r"\*/", Comment.Multiline, "#pop"),
+            (r"[^*]+", Comment.Multiline),
+        ],
+        # Handle numbers
+        "numerics": [
+            (r"\b\d+\.\d+(e\d+)?[fd]?\b", Number.Float),
+            (r"\b\d+\b", Number.Integer),
+        ],
+        # Handle variable names in things
+        "interp": [
+            (
+                r"(\$\()(\w+)(\))",
+                bygroups(String.Interpol, Name.Variable, String.Interpol),
+            ),
+        ],
+        # Handle strings
+        "string": [
+            (r"'", String, "#pop"),
+            include("interp"),
+            (r"[^'$]+", String),
+            (r"\$", String),
+        ],
+        #
+        "assignment": [
+            (r";", Punctuation, "#pop"),
+            include("root"),
+        ],
+        "field_name_quote": [
+            (r'"', String.Symbol, "#pop"),
+            include("interp"),
+            (r"[^\"$]+", String.Symbol),
+            (r"\$", String.Symbol),
+        ],
+        "field_name_bracket": [
+            (r"\]", String.Symbol, "#pop"),
+            include("interp"),
+            (r"[^\]$]+", String.Symbol),
+            (r"\$", String.Symbol),
+        ],
+        "function": [(r"\)", Punctuation, "#pop"), include("root")],
+        "root": [
+            # Whitespace and comments
+            (r"\s+", Text.Whitespace),
+            (r"/\*", Comment.Multiline, "comment"),
+            (r"//.*\n", Comment.Single),
+            # variable assignment
+            (r"(let|set)(\s+)", bygroups(Keyword.Declaration, Text.Whitespace),
+             "assignment"),
+            # Word operators
+            (words(OPERATORS_LIST["words"], prefix=r"\b", suffix=r"\b"),
+             Operator.Word),
+            # Statements
+            (words(STATEMENT_LIST, suffix=r"\b"), Keyword),
+            # Table names
+            (r"[a-z]\w*:", Keyword.Declaration),
+            # Constants
+            (words(CONSTANT_LIST, suffix=r"\b"), Keyword.Constant),
+            # Functions
+            (words(SCRIPT_FUNCTIONS, suffix=r"(?=\s*\()"), Name.Builtin,
+             "function"),
+            # interpolation - e.g. $(variableName)
+            include("interp"),
+            # Quotes denote a field/file name
+            (r'"', String.Symbol, "field_name_quote"),
+            # Square brackets denote a field/file name
+            (r"\[", String.Symbol, "field_name_bracket"),
+            # Strings
+            (r"'", String, "string"),
+            # Numbers
+            include("numerics"),
+            # Operator symbols
+            (words(OPERATORS_LIST["symbols"]), Operator),
+            # Strings denoted by single quotes
+            (r"'.+?'", String),
+            # Words as text
+            (r"\b\w+\b", Text),
+            # Basic punctuation
+            (r"[,;.()\\/]", Punctuation),
+        ],
+    }

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/special.py

@@ -0,0 +1,122 @@
+"""
+    pygments.lexers.special
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Special lexers.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import ast
+
+from pygments.lexer import Lexer, line_re
+from pygments.token import Token, Error, Text, Generic
+from pygments.util import get_choice_opt
+
+
+__all__ = ['TextLexer', 'OutputLexer', 'RawTokenLexer']
+
+
+class TextLexer(Lexer):
+    """
+    "Null" lexer, doesn't highlight anything.
+    """
+    name = 'Text only'
+    aliases = ['text']
+    filenames = ['*.txt']
+    mimetypes = ['text/plain']
+    url = ""
+    version_added = ''
+
+    priority = 0.01
+
+    def get_tokens_unprocessed(self, text):
+        yield 0, Text, text
+
+    def analyse_text(text):
+        return TextLexer.priority
+
+
+class OutputLexer(Lexer):
+    """
+    Simple lexer that highlights everything as ``Token.Generic.Output``.
+    """
+    name = 'Text output'
+    aliases = ['output']
+    url = ""
+    version_added = '2.10'
+    _example = "output/output"
+
+    def get_tokens_unprocessed(self, text):
+        yield 0, Generic.Output, text
+
+
+_ttype_cache = {}
+
+
+class RawTokenLexer(Lexer):
+    """
+    Recreate a token stream formatted with the `RawTokenFormatter`.
+
+    Additional options accepted:
+
+    `compress`
+        If set to ``"gz"`` or ``"bz2"``, decompress the token stream with
+        the given compression algorithm before lexing (default: ``""``).
+    """
+    name = 'Raw token data'
+    aliases = []
+    filenames = []
+    mimetypes = ['application/x-pygments-tokens']
+    url = 'https://pygments.org/docs/formatters/#RawTokenFormatter'
+    version_added = ''
+
+    def __init__(self, **options):
+        self.compress = get_choice_opt(options, 'compress',
+                                       ['', 'none', 'gz', 'bz2'], '')
+        Lexer.__init__(self, **options)
+
+    def get_tokens(self, text):
+        if self.compress:
+            if isinstance(text, str):
+                text = text.encode('latin1')
+            try:
+                if self.compress == 'gz':
+                    import gzip
+                    text = gzip.decompress(text)
+                elif self.compress == 'bz2':
+                    import bz2
+                    text = bz2.decompress(text)
+            except OSError:
+                yield Error, text.decode('latin1')
+        if isinstance(text, bytes):
+            text = text.decode('latin1')
+
+        # do not call Lexer.get_tokens() because stripping is not optional.
+        text = text.strip('\n') + '\n'
+        for i, t, v in self.get_tokens_unprocessed(text):
+            yield t, v
+
+    def get_tokens_unprocessed(self, text):
+        length = 0
+        for match in line_re.finditer(text):
+            try:
+                ttypestr, val = match.group().rstrip().split('\t', 1)
+                ttype = _ttype_cache.get(ttypestr)
+                if not ttype:
+                    ttype = Token
+                    ttypes = ttypestr.split('.')[1:]
+                    for ttype_ in ttypes:
+                        if not ttype_ or not ttype_[0].isupper():
+                            raise ValueError('malformed token name')
+                        ttype = getattr(ttype, ttype_)
+                    _ttype_cache[ttypestr] = ttype
+                val = ast.literal_eval(val)
+                if not isinstance(val, str):
+                    raise ValueError('expected str')
+            except (SyntaxError, ValueError):
+                val = match.group()
+                ttype = Error
+            yield length, ttype, val
+            length += len(val)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/tal.py

@@ -0,0 +1,77 @@
+"""
+    pygments.lexers.tal
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Uxntal
+
+    .. versionadded:: 2.12
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Keyword, Name, String, Number, \
+    Punctuation, Whitespace, Literal
+
+__all__ = ['TalLexer']
+
+
+class TalLexer(RegexLexer):
+    """
+    For Uxntal source code.
+    """
+
+    name = 'Tal'
+    aliases = ['tal', 'uxntal']
+    filenames = ['*.tal']
+    mimetypes = ['text/x-uxntal']
+    url = 'https://wiki.xxiivv.com/site/uxntal.html'
+    version_added = '2.12'
+
+    instructions = [
+        'BRK', 'LIT', 'INC', 'POP', 'DUP', 'NIP', 'SWP', 'OVR', 'ROT',
+        'EQU', 'NEQ', 'GTH', 'LTH', 'JMP', 'JCN', 'JSR', 'STH',
+        'LDZ', 'STZ', 'LDR', 'STR', 'LDA', 'STA', 'DEI', 'DEO',
+        'ADD', 'SUB', 'MUL', 'DIV', 'AND', 'ORA', 'EOR', 'SFT'
+    ]
+
+    tokens = {
+        # the comment delimiters must not be adjacent to non-space characters.
+        # this means ( foo ) is a valid comment but (foo) is not. this also
+        # applies to nested comments.
+        'comment': [
+            (r'(?<!\S)\((?!\S)', Comment.Multiline, '#push'), # nested comments
+            (r'(?<!\S)\)(?!\S)', Comment.Multiline, '#pop'), # nested comments
+            (r'[^()]+', Comment.Multiline), # comments
+            (r'[()]+', Comment.Multiline), # comments
+        ],
+        'root': [
+            (r'\s+', Whitespace), # spaces
+            (r'(?<!\S)\((?!\S)', Comment.Multiline, 'comment'), # comments
+            (words(instructions, prefix=r'(?<!\S)', suffix=r'2?k?r?(?!\S)'),
+             Keyword.Reserved), # instructions
+            (r'[][{}](?!\S)', Punctuation), # delimiters
+            (r'#([0-9a-f]{2}){1,2}(?!\S)', Number.Hex), # integer
+            (r'"\S+', String), # raw string
+            (r'([0-9a-f]{2}){1,2}(?!\S)', Literal), # raw integer
+            (r'[|$][0-9a-f]{1,4}(?!\S)', Keyword.Declaration), # abs/rel pad
+            (r'%\S+', Name.Decorator), # macro
+            (r'@\S+', Name.Function), # label
+            (r'&\S+', Name.Label), # sublabel
+            (r'/\S+', Name.Tag), # spacer
+            (r'\.\S+', Name.Variable.Magic), # literal zero page addr
+            (r',\S+', Name.Variable.Instance), # literal rel addr
+            (r';\S+', Name.Variable.Global), # literal abs addr
+            (r'-\S+', Literal), # raw zero page addr
+            (r'_\S+', Literal), # raw relative addr
+            (r'=\S+', Literal), # raw absolute addr
+            (r'!\S+', Name.Function), # immediate jump
+            (r'\?\S+', Name.Function), # conditional immediate jump
+            (r'~\S+', Keyword.Namespace), # include
+            (r'\S+', Name.Function), # macro invocation, immediate subroutine
+        ]
+    }
+
+    def analyse_text(text):
+        return '|0100' in text[:500]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/text.py

@@ -0,0 +1,27 @@
+"""
+    pygments.lexers.text
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for non-source code file types.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# ruff: noqa: F401
+from pygments.lexers.configs import ApacheConfLexer, NginxConfLexer, \
+    SquidConfLexer, LighttpdConfLexer, IniLexer, RegeditLexer, PropertiesLexer, \
+    UnixConfigLexer
+from pygments.lexers.console import PyPyLogLexer
+from pygments.lexers.textedit import VimLexer
+from pygments.lexers.markup import BBCodeLexer, MoinWikiLexer, RstLexer, \
+    TexLexer, GroffLexer
+from pygments.lexers.installers import DebianControlLexer, DebianSourcesLexer, SourcesListLexer
+from pygments.lexers.make import MakefileLexer, BaseMakefileLexer, CMakeLexer
+from pygments.lexers.haxe import HxmlLexer
+from pygments.lexers.sgf import SmartGameFormatLexer
+from pygments.lexers.diff import DiffLexer, DarcsPatchLexer
+from pygments.lexers.data import YamlLexer
+from pygments.lexers.textfmts import IrcLogsLexer, GettextLexer, HttpLexer
+
+__all__ = []

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/web.py

@@ -0,0 +1,24 @@
+"""
+    pygments.lexers.web
+    ~~~~~~~~~~~~~~~~~~~
+
+    Just export previously exported lexers.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# ruff: noqa: F401
+from pygments.lexers.html import HtmlLexer, DtdLexer, XmlLexer, XsltLexer, \
+    HamlLexer, ScamlLexer, JadeLexer
+from pygments.lexers.css import CssLexer, SassLexer, ScssLexer
+from pygments.lexers.javascript import JavascriptLexer, LiveScriptLexer, \
+    DartLexer, TypeScriptLexer, LassoLexer, ObjectiveJLexer, CoffeeScriptLexer
+from pygments.lexers.actionscript import ActionScriptLexer, \
+    ActionScript3Lexer, MxmlLexer
+from pygments.lexers.php import PhpLexer
+from pygments.lexers.webmisc import DuelLexer, XQueryLexer, SlimLexer, QmlLexer
+from pygments.lexers.data import JsonLexer
+JSONLexer = JsonLexer  # for backwards compatibility with Pygments 1.5
+
+__all__ = []

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/tokenizers-0.22.2.dist-info/RECORD

@@ -0,0 +1,31 @@
+tokenizers-0.22.2.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2
+tokenizers-0.22.2.dist-info/METADATA,sha256=FaXdr0ifWSt34Kk0wO60a1ETCpQGTKEpIyr9sKOGjvw,7254
+tokenizers-0.22.2.dist-info/RECORD,,
+tokenizers-0.22.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tokenizers-0.22.2.dist-info/WHEEL,sha256=5mwg5nCvp3YrLxikUrE5E0HBDKerMOoBBb70NjCncME,143
+tokenizers/__init__.py,sha256=FI7LEi8_7gO-mrsf4hPdhfvGkb8q0rQ3_1MVM3gaajo,2639
+tokenizers/__init__.pyi,sha256=MKWF2m4mz7IG1bPTdJ7AjXkQDNzkmQSLMmACQ2VUYJU,55891
+tokenizers/decoders/__init__.py,sha256=hfwM6CFUDvlMGGL4-xsaaYz81K9P5rQI5ZL5UHWK8Y4,372
+tokenizers/decoders/__init__.pyi,sha256=T60mFckMbS8YrsonOAPtfvb7VYHUJi9mm47Wd8pT62o,12019
+tokenizers/implementations/__init__.py,sha256=VzAsplaIo7rl4AFO8Miu7ig7MfZjvonwVblZw01zR6M,310
+tokenizers/implementations/base_tokenizer.py,sha256=PtQ2TSmoMGlTpL8oc8fDvwJVIY6isWGmps9comzsWjE,15806
+tokenizers/implementations/bert_wordpiece.py,sha256=sKCum0FKPYdSgJFJN8LDerVBoTDRSqyqSdrcm-lvQqI,5520
+tokenizers/implementations/byte_level_bpe.py,sha256=iBepM_z1s5Ky7zFDVrYLc3L5byYrIouk7-k0JGuF10s,4272
+tokenizers/implementations/char_level_bpe.py,sha256=Nag_HFq8Rvcucqi8MhV1-0xtoR0C7FjHOecFVURL7ss,5449
+tokenizers/implementations/sentencepiece_bpe.py,sha256=c08fKf6i92E2RsKgsxy7LzZfYX8-MACHSRG8U_I5ytY,3721
+tokenizers/implementations/sentencepiece_unigram.py,sha256=2RoIfFVpiMkJOtOCskM_VCeCELWaC_bNnds6GvtE0KQ,7630
+tokenizers/models/__init__.py,sha256=eJZ4HTAQZpxnKILNylWaTFqxXy-Ba6OKswWN47feeV8,176
+tokenizers/models/__init__.pyi,sha256=2gZPQR1Z5_krTzLXx-ts5ai7Fz7bTZ0QI1OSJ5MyOuc,19517
+tokenizers/normalizers/__init__.py,sha256=_06w4cqRItveEgIddYaLMScgkSOkIAMIzYCesb5AA4U,841
+tokenizers/normalizers/__init__.pyi,sha256=6zYmbFtvdF1WhoWQSdEN974mxHjc7ZwJBA0TI2dJk98,25709
+tokenizers/pre_tokenizers/__init__.py,sha256=KV9-EsAykGENUUzkGWCbv4n6YM6hYa1hfnY-gzBpMNE,598
+tokenizers/pre_tokenizers/__init__.pyi,sha256=_pc34-Kd2N7Nvs7vTHPULBKjm18iJRM9qLOClVHw9n4,31566
+tokenizers/processors/__init__.py,sha256=xM2DEKwKtHIumHsszM8AMkq-AlaqvBZFXWgLU8SNhOY,307
+tokenizers/processors/__init__.pyi,sha256=5L5OBZ7SXCg7AEy51jyDHViaCSHG5c7vW4eWjSVQbUs,14348
+tokenizers/tokenizers.abi3.so,sha256=wRb88egNRhzgo1wzKXTyWUnoNZQW9Qs9UzcYENLOHMw,10074176
+tokenizers/tokenizers.pyi,sha256=Mq4G5RcxKiVc0FZd_Omi-bT7YQMRc-iDBU_nPCmCZOA,468
+tokenizers/tools/__init__.py,sha256=xG8caB9OHC8cbB01S5vYV14HZxhO6eWbLehsb70ppio,55
+tokenizers/tools/visualizer-styles.css,sha256=zAydq1oGWD8QEll4-eyL8Llw0B1sty_hpIE3tYxL02k,4850
+tokenizers/tools/visualizer.py,sha256=jtxka01phNP47uQSocIQFO_DMnL3ZHdwohGVDqqYJPo,14834
+tokenizers/trainers/__init__.py,sha256=UTu22AGcp76IvpW45xLRbJWET04NxPW6NfCb2YYz0EM,248
+tokenizers/trainers/__init__.pyi,sha256=jKtDNXnoX6FWeCTeHz-W62Cj2_JErgYG7h1PReUz1rU,10719

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/tqdm/contrib/__init__.py

@@ -0,0 +1,91 @@
+"""
+Thin wrappers around common functions.
+
+Subpackages contain potentially unstable extensions.
+"""
+from warnings import warn
+
+from ..auto import tqdm as tqdm_auto
+from ..std import TqdmDeprecationWarning, tqdm
+from ..utils import ObjectWrapper
+
+__author__ = {"github.com/": ["casperdcl"]}
+__all__ = ['tenumerate', 'tzip', 'tmap']
+
+
+class DummyTqdmFile(ObjectWrapper):
+    """Dummy file-like that will write to tqdm"""
+
+    def __init__(self, wrapped):
+        super().__init__(wrapped)
+        self._buf = []
+
+    def write(self, x, nolock=False):
+        nl = b"\n" if isinstance(x, bytes) else "\n"
+        pre, sep, post = x.rpartition(nl)
+        if sep:
+            blank = type(nl)()
+            tqdm.write(blank.join(self._buf + [pre, sep]),
+                       end=blank, file=self._wrapped, nolock=nolock)
+            self._buf = [post]
+        else:
+            self._buf.append(x)
+
+    def __del__(self):
+        if self._buf:
+            blank = type(self._buf[0])()
+            try:
+                tqdm.write(blank.join(self._buf), end=blank, file=self._wrapped)
+            except (OSError, ValueError):
+                pass
+
+
+def builtin_iterable(func):
+    """Returns `func`"""
+    warn("This function has no effect, and will be removed in tqdm==5.0.0",
+         TqdmDeprecationWarning, stacklevel=2)
+    return func
+
+
+def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, **tqdm_kwargs):
+    """
+    Equivalent of `numpy.ndenumerate` or builtin `enumerate`.
+
+    Parameters
+    ----------
+    tqdm_class  : [default: tqdm.auto.tqdm].
+    """
+    try:
+        import numpy as np
+    except ImportError:
+        pass
+    else:
+        if isinstance(iterable, np.ndarray):
+            return tqdm_class(np.ndenumerate(iterable), total=total or iterable.size,
+                              **tqdm_kwargs)
+    return enumerate(tqdm_class(iterable, total=total, **tqdm_kwargs), start)
+
+
+def tzip(iter1, *iter2plus, **tqdm_kwargs):
+    """
+    Equivalent of builtin `zip`.
+
+    Parameters
+    ----------
+    tqdm_class  : [default: tqdm.auto.tqdm].
+    """
+    kwargs = tqdm_kwargs.copy()
+    tqdm_class = kwargs.pop("tqdm_class", tqdm_auto)
+    yield from zip(tqdm_class(iter1, **kwargs), *iter2plus)
+
+
+def tmap(function, *sequences, **tqdm_kwargs):
+    """
+    Equivalent of builtin `map`.
+
+    Parameters
+    ----------
+    tqdm_class  : [default: tqdm.auto.tqdm].
+    """
+    for i in tzip(*sequences, **tqdm_kwargs):
+        yield function(*i)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/tqdm/contrib/bells.py

@@ -0,0 +1,26 @@
+"""
+Even more features than `tqdm.auto` (all the bells & whistles):
+
+- `tqdm.auto`
+- `tqdm.tqdm.pandas`
+- `tqdm.contrib.telegram`
+    + uses `${TQDM_TELEGRAM_TOKEN}` and `${TQDM_TELEGRAM_CHAT_ID}`
+- `tqdm.contrib.discord`
+    + uses `${TQDM_DISCORD_TOKEN}` and `${TQDM_DISCORD_CHANNEL_ID}`
+"""
+__all__ = ['tqdm', 'trange']
+import warnings
+from os import getenv
+
+if getenv("TQDM_SLACK_TOKEN") and getenv("TQDM_SLACK_CHANNEL"):
+    from .slack import tqdm, trange
+elif getenv("TQDM_TELEGRAM_TOKEN") and getenv("TQDM_TELEGRAM_CHAT_ID"):
+    from .telegram import tqdm, trange
+elif getenv("TQDM_DISCORD_TOKEN") and getenv("TQDM_DISCORD_CHANNEL_ID"):
+    from .discord import tqdm, trange
+else:
+    from ..auto import tqdm, trange
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", category=FutureWarning)
+    tqdm.pandas()

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ernie4_5_moe/configuration_ernie4_5_moe.py

@@ -0,0 +1,121 @@
+# Copyright (c) 2025 Baidu, Inc. and HuggingFace Inc. team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Ernie 4.5 MoE model configuration"""
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...modeling_rope_utils import RopeParameters
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="baidu/ERNIE-4.5-21B-A3B-PT")
+@strict
+class Ernie4_5_MoeConfig(PreTrainedConfig):
+    r"""
+    use_bias (`bool`, *optional*, defaults to `False`):
+        Whether to use a bias in any of the projections including mlp and attention for example.
+    moe_k (`int`, *optional*, defaults to 6):
+        Number of selected experts.
+    moe_num_experts (`int`, *optional*, defaults to 64):
+        Number of routed experts.
+    moe_num_shared_experts (`int`, *optional*, defaults to 2):
+        The number of experts that are shared for all MoE forwards.
+    moe_layer_start_index (`int`, *optional*, defaults to 1):
+        The first index at which MoE layers start to appear.
+    moe_layer_end_index (`int`, *optional*, defaults to -1):
+        The last possible index for a MoE layer.
+    moe_layer_interval (`int`, *optional*, defaults to 1):
+        The intervals between MoE layers to appear.
+    moe_norm_min (`float`, *optional*, defaults to 1e-12):
+        Minimum division value during routing normalization.
+
+    Example:
+
+    ```python
+    >>> from transformers import Ernie4_5_MoeModel, Ernie4_5_MoEConfig
+
+    >>> # Initializing a Ernie4_5_MoE style configuration
+    >>> configuration = Ernie4_5_MoEConfig()
+
+    >>> # Initializing a model from the ERNIE-4.5-21B-A3B style configuration
+    >>> model = Ernie4_5_MoeModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "ernie4_5_moe"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {"num_experts": "moe_num_experts", "num_experts_per_tok": "moe_k"}
+    default_theta = 500000.0
+
+    # Default tensor parallel plan for base model `Ernie4_5_MoE`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.experts.gate_up_proj": "packed_colwise",
+        "layers.*.mlp.experts.down_proj": "rowwise",
+        "layers.*.mlp.experts": "moe_tp_experts",
+        "layers.*.mlp.shared_experts.gate_proj": "colwise",
+        "layers.*.mlp.shared_experts.up_proj": "colwise",
+        "layers.*.mlp.shared_experts.down_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    vocab_size: int = 103424
+    pad_token_id: int | None = 0
+    bos_token_id: int | None = 1
+    eos_token_id: int | list[int] | None = 2
+    hidden_size: int = 2560
+    intermediate_size: int = 12288
+    num_hidden_layers: int = 28
+    num_attention_heads: int = 20
+    num_key_value_heads: int | None = 4
+    hidden_act: str = "silu"
+    max_position_embeddings: int = 131072
+    initializer_range: float = 0.02
+    rms_norm_eps: float = 1e-5
+    use_cache: bool = True
+    tie_word_embeddings: bool = True
+    rope_parameters: RopeParameters | dict | None = None
+    use_bias: bool | None = False
+    moe_intermediate_size: int = 1536
+    moe_k: int | None = 6
+    moe_num_experts: int | None = 64
+    moe_num_shared_experts: int | None = 2
+    moe_layer_start_index: int | None = 1
+    moe_layer_end_index: int | None = -1
+    moe_layer_interval: int | None = 1
+    moe_norm_min: float | None = 1e-12
+    output_router_logits: bool | None = False
+    router_aux_loss_coef: float | None = 0.001
+
+    def __post_init__(self, **kwargs):
+        self.moe_layer_end_index = (
+            self.num_hidden_layers - 1 if self.moe_layer_end_index == -1 else self.moe_layer_end_index
+        )
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["Ernie4_5_MoeConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py

@@ -0,0 +1,742 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/ernie4_5_moe/modular_ernie4_5_moe.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_ernie4_5_moe.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright (c) 2025 Baidu, Inc. and HuggingFace Inc. team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Callable
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...cache_utils import Cache, DynamicCache
+from ...generation import GenerationMixin
+from ...integrations import use_experts_implementation, use_kernel_forward_from_hub, use_kernelized_func
+from ...masking_utils import create_causal_mask
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast
+from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, can_return_tuple
+from ...utils.generic import maybe_autocast, merge_with_config_defaults
+from ...utils.output_capturing import OutputRecorder, capture_outputs
+from .configuration_ernie4_5_moe import Ernie4_5_MoeConfig
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class Ernie4_5_MoeRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps: float = 1e-6) -> None:
+        """
+        Ernie4_5_MoeRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class Ernie4_5_MoeMLP(nn.Module):
+    def __init__(self, config, intermediate_size=None):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size
+
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.use_bias)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.use_bias)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.use_bias)
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x):
+        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        return down_proj
+
+
+class Ernie4_5_MoeRotaryEmbedding(nn.Module):
+    inv_freq: torch.Tensor  # fix linting for `register_buffer`
+
+    def __init__(self, config: Ernie4_5_MoeConfig, device=None):
+        super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+
+        self.config = config
+
+        self.rope_type = self.config.rope_parameters["rope_type"]
+        rope_init_fn: Callable = self.compute_default_rope_parameters
+        if self.rope_type != "default":
+            rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+        inv_freq, self.attention_scaling = rope_init_fn(self.config, device)
+
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
+
+    @staticmethod
+    def compute_default_rope_parameters(
+        config: Ernie4_5_MoeConfig | None = None,
+        device: Optional["torch.device"] = None,
+        seq_len: int | None = None,
+    ) -> tuple["torch.Tensor", float]:
+        """
+        Computes the inverse frequencies according to the original RoPE implementation
+        Args:
+            config ([`~transformers.PreTrainedConfig`]):
+                The model configuration.
+            device (`torch.device`):
+                The device to use for initialization of the inverse frequencies.
+            seq_len (`int`, *optional*):
+                The current sequence length. Unused for this type of RoPE.
+        Returns:
+            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
+        """
+        base = config.rope_parameters["rope_theta"]
+        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
+
+        attention_factor = 1.0  # Unused in this type of RoPE
+
+        # Compute the inverse frequencies
+        inv_freq = 1.0 / (
+            base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)
+        )
+        return inv_freq, attention_factor
+
+    @torch.no_grad()
+    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
+    def forward(self, x, position_ids):
+        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
+
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with maybe_autocast(device_type=device_type, enabled=False):  # Force float32
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
+
+        # keeping it in full precision
+        return cos, sin
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., 0::2]
+    x2 = x[..., 1::2]
+    return torch.stack((-x2, x1), dim=-1).flatten(-2)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    # glm rope style (with full dim) and full precision
+    original_dtype = q.dtype
+
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+
+    # Interleave them instead of usual shape
+    cos = cos[..., : cos.shape[-1] // 2].repeat_interleave(2, dim=-1)
+    sin = sin[..., : sin.shape[-1] // 2].repeat_interleave(2, dim=-1)
+
+    q_embed = (q.float() * cos) + (rotate_half(q).float() * sin)
+    k_embed = (k.float() * cos) + (rotate_half(k).float() * sin)
+
+    return q_embed.to(original_dtype), k_embed.to(original_dtype)
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    key_states = repeat_kv(key, module.num_key_value_groups)
+    value_states = repeat_kv(value, module.num_key_value_groups)
+
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+@use_kernelized_func(apply_rotary_pos_emb)
+class Ernie4_5_MoeAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: Ernie4_5_MoeConfig, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
+        self.scaling = self.head_dim**-0.5
+
+        self.attention_dropout = 0.0
+        self.is_causal = True
+
+        self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.use_bias)
+        self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.use_bias)
+        self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.use_bias)
+        self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.use_bias)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        attention_mask: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_values is not None:
+            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class Ernie4_5_MoeStatics(nn.Module):
+    """
+    Stores MoE (Mixture of Experts) statistics
+        - Bias for the gating
+        - Additionally, usage per expert in the original codebase
+    """
+
+    def __init__(self, config):
+        super().__init__()
+
+        num_experts_groups = 1
+        num_experts = config.moe_num_experts
+
+        self.e_score_correction_bias = nn.Parameter(
+            torch.zeros(num_experts_groups, num_experts, dtype=torch.float32),
+            requires_grad=False,
+        )
+
+    def forward(self, hidden_states):
+        # NOTE: This is a workaround to enable TP with a module that only has parameters
+        #
+        # Otherwise, it stays as `DTensor` when called in the "super" forward
+        #   1. All other tensors are local (`torch.Tensor`)
+        #   2. Isolate does not work on `nn.Module` which only has parameters
+        return hidden_states + self.e_score_correction_bias.squeeze()
+
+
+@use_experts_implementation
+class Ernie4_5_MoeExperts(nn.Module):
+    """Collection of expert weights stored as 3D tensors."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.num_experts = config.moe_num_experts
+        self.hidden_dim = config.hidden_size
+        self.intermediate_dim = config.moe_intermediate_size
+        self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim))
+        self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_dim, self.intermediate_dim))
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        top_k_index: torch.Tensor,
+        top_k_weights: torch.Tensor,
+    ) -> torch.Tensor:
+        final_hidden_states = torch.zeros_like(hidden_states)
+        with torch.no_grad():
+            expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)
+            expert_mask = expert_mask.permute(2, 1, 0)
+            expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()
+
+        for expert_idx in expert_hit:
+            expert_idx = expert_idx[0]
+            if expert_idx == self.num_experts:
+                continue
+            top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
+            current_state = hidden_states[token_idx]
+            gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)
+            current_hidden_states = self.act_fn(gate) * up
+            current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])
+            current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]
+            final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))
+
+        return final_hidden_states
+
+
+class Ernie4_5_MoeTopKRouter(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.weight = nn.Parameter(torch.zeros(config.moe_num_experts, config.hidden_size, dtype=torch.float32))
+        self.moe_statics = Ernie4_5_MoeStatics(config)
+        self.top_k = config.moe_k
+        self.norm_min = config.moe_norm_min
+
+    def forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        device_type = (
+            hidden_states.device.type
+            if isinstance(hidden_states.device.type, str) and hidden_states.device.type != "mps"
+            else "cpu"
+        )
+
+        with maybe_autocast(device_type=device_type, enabled=False):  # Force float32
+            router_logits = F.linear(hidden_states.float(), self.weight.float())
+            routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
+            _, selected_experts = torch.topk(self.moe_statics(routing_weights), self.top_k, dim=-1)
+            routing_weights = torch.gather(routing_weights, dim=-1, index=selected_experts)
+            routing_weights = routing_weights / torch.clamp(
+                routing_weights.sum(dim=-1, keepdim=True), min=self.norm_min
+            )
+        routing_weights = routing_weights.to(hidden_states.dtype)
+        return router_logits, selected_experts, routing_weights
+
+
+class Ernie4_5_MoeSparseMoeBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.hidden_dim = config.hidden_size
+        self.num_experts = config.moe_num_experts
+        self.top_k = config.moe_k
+        self.gate = Ernie4_5_MoeTopKRouter(config)
+        self.experts = Ernie4_5_MoeExperts(config)
+
+        self.shared_experts = None
+        if config.moe_num_shared_experts > 0:
+            self.shared_experts = Ernie4_5_MoeMLP(config, config.moe_intermediate_size * config.moe_num_shared_experts)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        batch_size, sequence_length, _ = hidden_states.shape
+        hidden_states = hidden_states.view(-1, self.hidden_dim)
+
+        if self.shared_experts is not None:
+            shared_output = self.shared_experts(hidden_states)
+
+        _, top_k_index, top_k_weights = self.gate(hidden_states)
+        final_hidden_states = self.experts(hidden_states, top_k_index, top_k_weights)
+
+        if self.shared_experts is not None:
+            final_hidden_states = final_hidden_states + shared_output
+
+        final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, self.hidden_dim)
+        return final_hidden_states.to(hidden_states.dtype)
+
+
+class Ernie4_5_MoeDecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config, layer_idx):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+
+        self.self_attn = Ernie4_5_MoeAttention(config, layer_idx)
+
+        if (
+            ((layer_idx + 1) % config.moe_layer_interval == 0)
+            and layer_idx >= config.moe_layer_start_index
+            and layer_idx <= config.moe_layer_end_index
+        ):
+            self.mlp = Ernie4_5_MoeSparseMoeBlock(config)
+        else:
+            self.mlp = Ernie4_5_MoeMLP(config)
+
+        self.input_layernorm = Ernie4_5_MoeRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.post_attention_layernorm = Ernie4_5_MoeRMSNorm(config.hidden_size, config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        use_cache: bool | None = False,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        # Self Attention
+        hidden_states, _ = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            position_embeddings=position_embeddings,
+            **kwargs,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        return hidden_states
+
+
+@auto_docstring
+class Ernie4_5_MoePreTrainedModel(PreTrainedModel):
+    config: Ernie4_5_MoeConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Ernie4_5_MoeDecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+
+    _can_compile_fullgraph = True
+    _supports_attention_backend = True
+    _can_record_outputs = {
+        "router_logits": OutputRecorder(Ernie4_5_MoeTopKRouter, index=0),
+        "hidden_states": Ernie4_5_MoeDecoderLayer,
+        "attentions": Ernie4_5_MoeAttention,
+    }
+    # Not supporting multi-token prediction (MTP) atm
+    _keys_to_ignore_on_load_unexpected = ["mtp"]
+    _keep_in_fp32_modules_strict = ["gate.weight", "moe_statics"]
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, Ernie4_5_MoeStatics):
+            init.zeros_(module.e_score_correction_bias)
+        elif isinstance(module, Ernie4_5_MoeExperts):
+            init.normal_(module.gate_up_proj, mean=0.0, std=self.config.initializer_range)
+            init.normal_(module.down_proj, mean=0.0, std=self.config.initializer_range)
+
+
+@auto_docstring
+class Ernie4_5_MoeModel(Ernie4_5_MoePreTrainedModel):
+    def __init__(self, config: Ernie4_5_MoeConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [Ernie4_5_MoeDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.norm = Ernie4_5_MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = Ernie4_5_MoeRotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> MoeModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if position_ids is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
+            position_ids = position_ids.unsqueeze(0)
+
+        causal_mask = create_causal_mask(
+            config=self.config,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            position_ids=position_ids,
+        )
+
+        hidden_states = inputs_embeds
+
+        # create position embeddings to be shared across the decoder layers
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+            hidden_states = decoder_layer(
+                hidden_states,
+                position_embeddings=position_embeddings,
+                attention_mask=causal_mask,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                use_cache=use_cache,
+                **kwargs,
+            )
+
+        hidden_states = self.norm(hidden_states)
+
+        return MoeModelOutputWithPast(  # only diff with Mistral is the output type, we need MoE
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values,
+        )
+
+
+def load_balancing_loss_func(
+    gate_logits: torch.Tensor | tuple[torch.Tensor] | None,
+    num_experts: int | None = None,
+    top_k=2,
+    attention_mask: torch.Tensor | None = None,
+) -> torch.Tensor | int:
+    r"""
+    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.
+
+    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
+    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
+    experts is too unbalanced.
+
+    Args:
+        gate_logits:
+            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
+            shape [batch_size X sequence_length, num_experts].
+        num_experts:
+            Number of experts
+        top_k:
+            The number of experts to route per-token, can be also interpreted as the `top-k` routing
+            parameter.
+        attention_mask (`torch.Tensor`, *optional*):
+            The attention_mask used in forward function
+            shape [batch_size X sequence_length] if not None.
+
+    Returns:
+        The auxiliary loss.
+    """
+    if gate_logits is None or not isinstance(gate_logits, tuple):
+        return 0
+
+    if isinstance(gate_logits, tuple):
+        compute_device = gate_logits[0].device
+        concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0)
+
+    routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1)
+
+    _, selected_experts = torch.topk(routing_weights, top_k, dim=-1)
+
+    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts)
+
+    if attention_mask is None:
+        # Compute the percentage of tokens routed to each experts
+        tokens_per_expert = torch.mean(expert_mask.float(), dim=0)
+
+        # Compute the average probability of routing to these experts
+        router_prob_per_expert = torch.mean(routing_weights, dim=0)
+    else:
+        batch_size, sequence_length = attention_mask.shape
+        num_hidden_layers = concatenated_gate_logits.shape[0] // (batch_size * sequence_length)
+
+        # Compute the mask that masks all padding tokens as 0 with the same shape of expert_mask
+        expert_attention_mask = (
+            attention_mask[None, :, :, None, None]
+            .expand((num_hidden_layers, batch_size, sequence_length, top_k, num_experts))
+            .reshape(-1, top_k, num_experts)
+            .to(compute_device)
+        )
+
+        # Compute the percentage of tokens routed to each experts
+        tokens_per_expert = torch.sum(expert_mask.float() * expert_attention_mask, dim=0) / torch.sum(
+            expert_attention_mask, dim=0
+        )
+
+        # Compute the mask that masks all padding tokens as 0 with the same shape of tokens_per_expert
+        router_per_expert_attention_mask = (
+            attention_mask[None, :, :, None]
+            .expand((num_hidden_layers, batch_size, sequence_length, num_experts))
+            .reshape(-1, num_experts)
+            .to(compute_device)
+        )
+
+        # Compute the average probability of routing to these experts
+        router_prob_per_expert = torch.sum(routing_weights * router_per_expert_attention_mask, dim=0) / torch.sum(
+            router_per_expert_attention_mask, dim=0
+        )
+
+    overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0))
+    return overall_loss * num_experts
+
+
+@auto_docstring
+class Ernie4_5_MoeForCausalLM(Ernie4_5_MoePreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+    _tp_plan = {"lm_head": "colwise_gather_output"}
+    _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = Ernie4_5_MoeModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=config.use_bias)
+
+        self.router_aux_loss_coef = config.router_aux_loss_coef
+        self.num_experts = config.moe_num_experts
+        self.num_experts_per_tok = config.moe_k
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        output_router_logits: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> MoeCausalLMOutputWithPast:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+        """
+
+        output_router_logits = (
+            output_router_logits if output_router_logits is not None else self.config.output_router_logits
+        )
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs: MoeModelOutputWithPast = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_router_logits=output_router_logits,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits, labels, self.vocab_size, **kwargs)
+
+        aux_loss = None
+        if output_router_logits:
+            aux_loss = load_balancing_loss_func(
+                outputs.router_logits,
+                self.num_experts,
+                self.num_experts_per_tok,
+                attention_mask,
+            )
+            if labels is not None:
+                loss += self.router_aux_loss_coef * aux_loss.to(loss.device)  # make sure to reside in the same device
+
+        return MoeCausalLMOutputWithPast(
+            loss=loss,
+            aux_loss=aux_loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            router_logits=outputs.router_logits,
+        )
+
+
+__all__ = ["Ernie4_5_MoeForCausalLM", "Ernie4_5_MoeModel", "Ernie4_5_MoePreTrainedModel"]