Add files using upload-large-folder tool

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/_compat.py

@@ -0,0 +1 @@
+from __future__ import annotations

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/main.py

@@ -0,0 +1,351 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Generator, Iterable, Mapping, MutableMapping
+from contextlib import contextmanager
+from typing import Any, Literal, overload
+
+from . import helpers, presets
+from .common import normalize_url, utils
+from .parser_block import ParserBlock
+from .parser_core import ParserCore
+from .parser_inline import ParserInline
+from .renderer import RendererHTML, RendererProtocol
+from .rules_core.state_core import StateCore
+from .token import Token
+from .utils import EnvType, OptionsDict, OptionsType, PresetType
+
+try:
+    import linkify_it
+except ModuleNotFoundError:
+    linkify_it = None
+
+
+_PRESETS: dict[str, PresetType] = {
+    "default": presets.default.make(),
+    "js-default": presets.js_default.make(),
+    "zero": presets.zero.make(),
+    "commonmark": presets.commonmark.make(),
+    "gfm-like": presets.gfm_like.make(),
+    "gfm-like2": presets.gfm_like2.make(),
+}
+
+
+class MarkdownIt:
+    def __init__(
+        self,
+        config: str | PresetType = "commonmark",
+        options_update: Mapping[str, Any] | None = None,
+        *,
+        renderer_cls: Callable[[MarkdownIt], RendererProtocol] = RendererHTML,
+    ):
+        """Main parser class
+
+        :param config: name of configuration to load or a pre-defined dictionary
+        :param options_update: dictionary that will be merged into ``config["options"]``
+        :param renderer_cls: the class to load as the renderer:
+            ``self.renderer = renderer_cls(self)
+        """
+        # add modules
+        self.utils = utils
+        self.helpers = helpers
+
+        # initialise classes
+        self.inline = ParserInline()
+        self.block = ParserBlock()
+        self.core = ParserCore()
+        self.renderer = renderer_cls(self)
+        self.linkify = linkify_it.LinkifyIt() if linkify_it else None
+
+        # set the configuration
+        if options_update and not isinstance(options_update, Mapping):
+            # catch signature change where renderer_cls was not used as a key-word
+            raise TypeError(
+                f"options_update should be a mapping: {options_update}"
+                "\n(Perhaps you intended this to be the renderer_cls?)"
+            )
+        self.configure(config, options_update=options_update)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__module__}.{self.__class__.__name__}()"
+
+    @overload
+    def __getitem__(self, name: Literal["inline"]) -> ParserInline: ...
+
+    @overload
+    def __getitem__(self, name: Literal["block"]) -> ParserBlock: ...
+
+    @overload
+    def __getitem__(self, name: Literal["core"]) -> ParserCore: ...
+
+    @overload
+    def __getitem__(self, name: Literal["renderer"]) -> RendererProtocol: ...
+
+    @overload
+    def __getitem__(self, name: str) -> Any: ...
+
+    def __getitem__(self, name: str) -> Any:
+        return {
+            "inline": self.inline,
+            "block": self.block,
+            "core": self.core,
+            "renderer": self.renderer,
+        }[name]
+
+    def set(self, options: OptionsType) -> None:
+        """Set parser options (in the same format as in constructor).
+        Probably, you will never need it, but you can change options after constructor call.
+
+        __Note:__ To achieve the best possible performance, don't modify a
+        `markdown-it` instance options on the fly. If you need multiple configurations
+        it's best to create multiple instances and initialize each with separate config.
+        """
+        self.options = OptionsDict(options)
+
+    def configure(
+        self, presets: str | PresetType, options_update: Mapping[str, Any] | None = None
+    ) -> MarkdownIt:
+        """Batch load of all options and component settings.
+        This is an internal method, and you probably will not need it.
+        But if you will - see available presets and data structure
+        [here](https://github.com/markdown-it/markdown-it/tree/master/lib/presets)
+
+        We strongly recommend to use presets instead of direct config loads.
+        That will give better compatibility with next versions.
+        """
+        if isinstance(presets, str):
+            if presets not in _PRESETS:
+                raise KeyError(f"Wrong `markdown-it` preset '{presets}', check name")
+            config = _PRESETS[presets]
+        else:
+            config = presets
+
+        if not config:
+            raise ValueError("Wrong `markdown-it` config, can't be empty")
+
+        options = config.get("options", {}) or {}
+        if options_update:
+            options = {**options, **options_update}  # type: ignore
+
+        self.set(options)
+
+        if "components" in config:
+            for name, component in config["components"].items():
+                rules = component.get("rules", None)
+                if rules:
+                    self[name].ruler.enableOnly(rules)
+                rules2 = component.get("rules2", None)
+                if rules2:
+                    self[name].ruler2.enableOnly(rules2)
+
+        return self
+
+    def get_all_rules(self) -> dict[str, list[str]]:
+        """Return the names of all active rules."""
+        rules = {
+            chain: self[chain].ruler.get_all_rules()
+            for chain in ["core", "block", "inline"]
+        }
+        rules["inline2"] = self.inline.ruler2.get_all_rules()
+        return rules
+
+    def get_active_rules(self) -> dict[str, list[str]]:
+        """Return the names of all active rules."""
+        rules = {
+            chain: self[chain].ruler.get_active_rules()
+            for chain in ["core", "block", "inline"]
+        }
+        rules["inline2"] = self.inline.ruler2.get_active_rules()
+        return rules
+
+    def enable(
+        self, names: str | Iterable[str], ignoreInvalid: bool = False
+    ) -> MarkdownIt:
+        """Enable list or rules. (chainable)
+
+        :param names: rule name or list of rule names to enable.
+        :param ignoreInvalid: set `true` to ignore errors when rule not found.
+
+        It will automatically find appropriate components,
+        containing rules with given names. If rule not found, and `ignoreInvalid`
+        not set - throws exception.
+
+        Example::
+
+            md = MarkdownIt().enable(['sub', 'sup']).disable('smartquotes')
+
+        """
+        result = []
+
+        if isinstance(names, str):
+            names = [names]
+
+        for chain in ["core", "block", "inline"]:
+            result.extend(self[chain].ruler.enable(names, True))
+        result.extend(self.inline.ruler2.enable(names, True))
+
+        missed = [name for name in names if name not in result]
+        if missed and not ignoreInvalid:
+            raise ValueError(f"MarkdownIt. Failed to enable unknown rule(s): {missed}")
+
+        return self
+
+    def disable(
+        self, names: str | Iterable[str], ignoreInvalid: bool = False
+    ) -> MarkdownIt:
+        """The same as [[MarkdownIt.enable]], but turn specified rules off. (chainable)
+
+        :param names: rule name or list of rule names to disable.
+        :param ignoreInvalid: set `true` to ignore errors when rule not found.
+
+        """
+        result = []
+
+        if isinstance(names, str):
+            names = [names]
+
+        for chain in ["core", "block", "inline"]:
+            result.extend(self[chain].ruler.disable(names, True))
+        result.extend(self.inline.ruler2.disable(names, True))
+
+        missed = [name for name in names if name not in result]
+        if missed and not ignoreInvalid:
+            raise ValueError(f"MarkdownIt. Failed to disable unknown rule(s): {missed}")
+        return self
+
+    @contextmanager
+    def reset_rules(self) -> Generator[None, None, None]:
+        """A context manager, that will reset the current enabled rules on exit."""
+        chain_rules = self.get_active_rules()
+        yield
+        for chain, rules in chain_rules.items():
+            if chain != "inline2":
+                self[chain].ruler.enableOnly(rules)
+        self.inline.ruler2.enableOnly(chain_rules["inline2"])
+
+    def add_render_rule(
+        self, name: str, function: Callable[..., Any], fmt: str = "html"
+    ) -> None:
+        """Add a rule for rendering a particular Token type.
+
+        Only applied when ``renderer.__output__ == fmt``
+        """
+        if self.renderer.__output__ == fmt:
+            self.renderer.rules[name] = function.__get__(self.renderer)  # type: ignore
+
+    def use(
+        self, plugin: Callable[..., None], *params: Any, **options: Any
+    ) -> MarkdownIt:
+        """Load specified plugin with given params into current parser instance. (chainable)
+
+        It's just a sugar to call `plugin(md, params)` with curring.
+
+        Example::
+
+            def func(tokens, idx):
+                tokens[idx].content = tokens[idx].content.replace('foo', 'bar')
+            md = MarkdownIt().use(plugin, 'foo_replace', 'text', func)
+
+        """
+        plugin(self, *params, **options)
+        return self
+
+    def parse(self, src: str, env: EnvType | None = None) -> list[Token]:
+        """Parse the source string to a token stream
+
+        :param src: source string
+        :param env: environment sandbox
+
+        Parse input string and return list of block tokens (special token type
+        "inline" will contain list of inline tokens).
+
+        `env` is used to pass data between "distributed" rules and return additional
+        metadata like reference info, needed for the renderer. It also can be used to
+        inject data in specific cases. Usually, you will be ok to pass `{}`,
+        and then pass updated object to renderer.
+        """
+        env = {} if env is None else env
+        if not isinstance(env, MutableMapping):
+            raise TypeError(f"Input data should be a MutableMapping, not {type(env)}")
+        if not isinstance(src, str):
+            raise TypeError(f"Input data should be a string, not {type(src)}")
+        state = StateCore(src, self, env)
+        self.core.process(state)
+        return state.tokens
+
+    def render(self, src: str, env: EnvType | None = None) -> Any:
+        """Render markdown string into html. It does all magic for you :).
+
+        :param src: source string
+        :param env: environment sandbox
+        :returns: The output of the loaded renderer
+
+        `env` can be used to inject additional metadata (`{}` by default).
+        But you will not need it with high probability. See also comment
+        in [[MarkdownIt.parse]].
+        """
+        env = {} if env is None else env
+        return self.renderer.render(self.parse(src, env), self.options, env)
+
+    def parseInline(self, src: str, env: EnvType | None = None) -> list[Token]:
+        """The same as [[MarkdownIt.parse]] but skip all block rules.
+
+        :param src: source string
+        :param env: environment sandbox
+
+        It returns the
+        block tokens list with the single `inline` element, containing parsed inline
+        tokens in `children` property. Also updates `env` object.
+        """
+        env = {} if env is None else env
+        if not isinstance(env, MutableMapping):
+            raise TypeError(f"Input data should be an MutableMapping, not {type(env)}")
+        if not isinstance(src, str):
+            raise TypeError(f"Input data should be a string, not {type(src)}")
+        state = StateCore(src, self, env)
+        state.inlineMode = True
+        self.core.process(state)
+        return state.tokens
+
+    def renderInline(self, src: str, env: EnvType | None = None) -> Any:
+        """Similar to [[MarkdownIt.render]] but for single paragraph content.
+
+        :param src: source string
+        :param env: environment sandbox
+
+        Similar to [[MarkdownIt.render]] but for single paragraph content. Result
+        will NOT be wrapped into `<p>` tags.
+        """
+        env = {} if env is None else env
+        return self.renderer.render(self.parseInline(src, env), self.options, env)
+
+    # link methods
+
+    def validateLink(self, url: str) -> bool:
+        """Validate if the URL link is allowed in output.
+
+        This validator can prohibit more than really needed to prevent XSS.
+        It's a tradeoff to keep code simple and to be secure by default.
+
+        Note: the url should be normalized at this point, and existing entities decoded.
+        """
+        return normalize_url.validateLink(url)
+
+    def normalizeLink(self, url: str) -> str:
+        """Normalize destination URLs in links
+
+        ::
+
+            [label]:   destination   'title'
+                    ^^^^^^^^^^^
+        """
+        return normalize_url.normalizeLink(url)
+
+    def normalizeLinkText(self, link: str) -> str:
+        """Normalize autolink content
+
+        ::
+
+            <destination>
+            ~~~~~~~~~~~
+        """
+        return normalize_url.normalizeLinkText(link)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/parser_block.py

@@ -0,0 +1,113 @@
+"""Block-level tokenizer."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+import logging
+from typing import TYPE_CHECKING
+
+from . import rules_block
+from .ruler import Ruler
+from .rules_block.state_block import StateBlock
+from .token import Token
+from .utils import EnvType
+
+if TYPE_CHECKING:
+    from markdown_it import MarkdownIt
+
+LOGGER = logging.getLogger(__name__)
+
+
+RuleFuncBlockType = Callable[[StateBlock, int, int, bool], bool]
+"""(state: StateBlock, startLine: int, endLine: int, silent: bool) -> matched: bool)
+
+`silent` disables token generation, useful for lookahead.
+"""
+
+_rules: list[tuple[str, RuleFuncBlockType, list[str]]] = [
+    # First 2 params - rule name & source. Secondary array - list of rules,
+    # which can be terminated by this one.
+    ("table", rules_block.table, ["paragraph", "reference"]),
+    ("code", rules_block.code, []),
+    ("fence", rules_block.fence, ["paragraph", "reference", "blockquote", "list"]),
+    (
+        "blockquote",
+        rules_block.blockquote,
+        ["paragraph", "reference", "blockquote", "list"],
+    ),
+    ("hr", rules_block.hr, ["paragraph", "reference", "blockquote", "list"]),
+    ("list", rules_block.list_block, ["paragraph", "reference", "blockquote"]),
+    ("reference", rules_block.reference, []),
+    ("html_block", rules_block.html_block, ["paragraph", "reference", "blockquote"]),
+    ("heading", rules_block.heading, ["paragraph", "reference", "blockquote"]),
+    ("lheading", rules_block.lheading, []),
+    ("paragraph", rules_block.paragraph, []),
+]
+
+
+class ParserBlock:
+    """
+    ParserBlock#ruler -> Ruler
+
+    [[Ruler]] instance. Keep configuration of block rules.
+    """
+
+    def __init__(self) -> None:
+        self.ruler = Ruler[RuleFuncBlockType]()
+        for name, rule, alt in _rules:
+            self.ruler.push(name, rule, {"alt": alt})
+
+    def tokenize(self, state: StateBlock, startLine: int, endLine: int) -> None:
+        """Generate tokens for input range."""
+        rules = self.ruler.getRules("")
+        line = startLine
+        maxNesting = state.md.options.maxNesting
+        hasEmptyLines = False
+
+        while line < endLine:
+            state.line = line = state.skipEmptyLines(line)
+            if line >= endLine:
+                break
+            if state.sCount[line] < state.blkIndent:
+                # Termination condition for nested calls.
+                # Nested calls currently used for blockquotes & lists
+                break
+            if state.level >= maxNesting:
+                # If nesting level exceeded - skip tail to the end.
+                # That's not ordinary situation and we should not care about content.
+                state.line = endLine
+                break
+
+            # Try all possible rules.
+            # On success, rule should:
+            # - update `state.line`
+            # - update `state.tokens`
+            # - return True
+            for rule in rules:
+                if rule(state, line, endLine, False):
+                    break
+
+            # set state.tight if we had an empty line before current tag
+            # i.e. latest empty line should not count
+            state.tight = not hasEmptyLines
+
+            line = state.line
+
+            # paragraph might "eat" one newline after it in nested lists
+            if (line - 1) < endLine and state.isEmpty(line - 1):
+                hasEmptyLines = True
+
+            if line < endLine and state.isEmpty(line):
+                hasEmptyLines = True
+                line += 1
+                state.line = line
+
+    def parse(
+        self, src: str, md: MarkdownIt, env: EnvType, outTokens: list[Token]
+    ) -> list[Token] | None:
+        """Process input string and push block tokens into `outTokens`."""
+        if not src:
+            return None
+        state = StateBlock(src, md, env, outTokens)
+        self.tokenize(state, state.line, state.lineMax)
+        return state.tokens

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/parser_core.py

@@ -0,0 +1,46 @@
+"""
+* class Core
+*
+* Top-level rules executor. Glues block/inline parsers and does intermediate
+* transformations.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from .ruler import Ruler
+from .rules_core import (
+    block,
+    inline,
+    linkify,
+    normalize,
+    replace,
+    smartquotes,
+    text_join,
+)
+from .rules_core.state_core import StateCore
+
+RuleFuncCoreType = Callable[[StateCore], None]
+
+_rules: list[tuple[str, RuleFuncCoreType]] = [
+    ("normalize", normalize),
+    ("block", block),
+    ("inline", inline),
+    ("linkify", linkify),
+    ("replacements", replace),
+    ("smartquotes", smartquotes),
+    ("text_join", text_join),
+]
+
+
+class ParserCore:
+    def __init__(self) -> None:
+        self.ruler = Ruler[RuleFuncCoreType]()
+        for name, rule in _rules:
+            self.ruler.push(name, rule)
+
+    def process(self, state: StateCore) -> None:
+        """Executes core chain rules."""
+        for rule in self.ruler.getRules(""):
+            rule(state)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/parser_inline.py

@@ -0,0 +1,215 @@
+"""Tokenizes paragraph content."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+import functools
+import re
+from typing import TYPE_CHECKING
+
+from . import rules_inline
+from .ruler import Ruler
+from .rules_inline.state_inline import StateInline
+from .token import Token
+from .utils import EnvType
+
+if TYPE_CHECKING:
+    from markdown_it import MarkdownIt
+
+
+# Default set of characters that terminate a text token and allow inline rules to fire.
+# '{}$%@~+=:' reserved for extensions.
+# Note: Don't confuse with "Markdown ASCII Punctuation" chars.
+# http://spec.commonmark.org/0.15/#ascii-punctuation-character
+_DEFAULT_TERMINATORS: frozenset[str] = frozenset(
+    {
+        "\n",
+        "!",
+        "#",
+        "$",
+        "%",
+        "&",
+        "*",
+        "+",
+        "-",
+        ":",
+        "<",
+        "=",
+        ">",
+        "@",
+        "[",
+        "\\",
+        "]",
+        "^",
+        "_",
+        "`",
+        "{",
+        "}",
+        "~",
+    }
+)
+
+
+# Lazily compiled regex for the default terminator set.  The @cache ensures it is
+# compiled at most once (on first ParserInline instantiation) and shared across all
+# instances that have not added extra chars, keeping __init__ cost near zero.
+@functools.cache
+def _default_terminator_re() -> re.Pattern[str]:
+    return re.compile("[" + re.escape("".join(_DEFAULT_TERMINATORS)) + "]")
+
+
+# Parser rules
+RuleFuncInlineType = Callable[[StateInline, bool], bool]
+"""(state: StateInline, silent: bool) -> matched: bool)
+
+`silent` disables token generation, useful for lookahead.
+"""
+_rules: list[tuple[str, RuleFuncInlineType]] = [
+    ("text", rules_inline.text),
+    ("linkify", rules_inline.linkify),
+    ("newline", rules_inline.newline),
+    ("escape", rules_inline.escape),
+    ("backticks", rules_inline.backtick),
+    ("strikethrough", rules_inline.strikethrough.tokenize),
+    ("emphasis", rules_inline.emphasis.tokenize),
+    ("link", rules_inline.link),
+    ("image", rules_inline.image),
+    ("autolink", rules_inline.autolink),
+    ("html_inline", rules_inline.html_inline),
+    ("entity", rules_inline.entity),
+]
+
+# Note `rule2` ruleset was created specifically for emphasis/strikethrough
+# post-processing and may be changed in the future.
+#
+# Don't use this for anything except pairs (plugins working with `balance_pairs`).
+#
+RuleFuncInline2Type = Callable[[StateInline], None]
+_rules2: list[tuple[str, RuleFuncInline2Type]] = [
+    ("balance_pairs", rules_inline.link_pairs),
+    ("strikethrough", rules_inline.strikethrough.postProcess),
+    ("emphasis", rules_inline.emphasis.postProcess),
+    # rules for pairs separate '**' into its own text tokens, which may be left unused,
+    # rule below merges unused segments back with the rest of the text
+    ("fragments_join", rules_inline.fragments_join),
+]
+
+
+class ParserInline:
+    def __init__(self) -> None:
+        self.ruler = Ruler[RuleFuncInlineType]()
+        for name, rule in _rules:
+            self.ruler.push(name, rule)
+        # Second ruler used for post-processing (e.g. in emphasis-like rules)
+        self.ruler2 = Ruler[RuleFuncInline2Type]()
+        for name, rule2 in _rules2:
+            self.ruler2.push(name, rule2)
+        # Characters that stop the text rule, allowing other inline rules to fire.
+        # _extra_terminator_chars is only allocated when add_terminator_char() is called
+        # with a char outside the defaults, keeping __init__ allocation-free.
+        self._extra_terminator_chars: set[str] = set()
+        # Pre-compiled regex shared with all default instances (no copy in the common path).
+        self.terminator_re: re.Pattern[str] = _default_terminator_re()
+
+    def add_terminator_char(self, ch: str) -> None:
+        """Register a character that stops the ``text`` rule, allowing inline rules to fire.
+
+        This lets plugins declare which characters their inline rules react to,
+        mirroring the ``MARKER`` mechanism in the Rust markdown-it implementation.
+
+        :param ch: A single character to add to the terminator set.
+        """
+        if ch not in _DEFAULT_TERMINATORS and ch not in self._extra_terminator_chars:
+            self._extra_terminator_chars.add(ch)
+            self.terminator_re = re.compile(
+                "["
+                + re.escape(
+                    "".join(_DEFAULT_TERMINATORS | self._extra_terminator_chars)
+                )
+                + "]"
+            )
+
+    def skipToken(self, state: StateInline) -> None:
+        """Skip single token by running all rules in validation mode;
+        returns `True` if any rule reported success
+        """
+        ok = False
+        pos = state.pos
+        rules = self.ruler.getRules("")
+        maxNesting = state.md.options["maxNesting"]
+        cache = state.cache
+
+        if pos in cache:
+            state.pos = cache[pos]
+            return
+
+        if state.level < maxNesting:
+            for rule in rules:
+                #  Increment state.level and decrement it later to limit recursion.
+                # It's harmless to do here, because no tokens are created.
+                # But ideally, we'd need a separate private state variable for this purpose.
+                state.level += 1
+                ok = rule(state, True)
+                state.level -= 1
+                if ok:
+                    break
+        else:
+            # Too much nesting, just skip until the end of the paragraph.
+            #
+            # NOTE: this will cause links to behave incorrectly in the following case,
+            #       when an amount of `[` is exactly equal to `maxNesting + 1`:
+            #
+            #       [[[[[[[[[[[[[[[[[[[[[foo]()
+            #
+            # TODO: remove this workaround when CM standard will allow nested links
+            #       (we can replace it by preventing links from being parsed in
+            #       validation mode)
+            #
+            state.pos = state.posMax
+
+        if not ok:
+            state.pos += 1
+        cache[pos] = state.pos
+
+    def tokenize(self, state: StateInline) -> None:
+        """Generate tokens for input range."""
+        ok = False
+        rules = self.ruler.getRules("")
+        end = state.posMax
+        maxNesting = state.md.options["maxNesting"]
+
+        while state.pos < end:
+            # Try all possible rules.
+            # On success, rule should:
+            #
+            # - update `state.pos`
+            # - update `state.tokens`
+            # - return true
+
+            if state.level < maxNesting:
+                for rule in rules:
+                    ok = rule(state, False)
+                    if ok:
+                        break
+
+            if ok:
+                if state.pos >= end:
+                    break
+                continue
+
+            state.pending += state.src[state.pos]
+            state.pos += 1
+
+        if state.pending:
+            state.pushPending()
+
+    def parse(
+        self, src: str, md: MarkdownIt, env: EnvType, tokens: list[Token]
+    ) -> list[Token]:
+        """Process input string and push inline tokens into `tokens`"""
+        state = StateInline(src, md, env, tokens)
+        self.tokenize(state)
+        rules2 = self.ruler2.getRules("")
+        for rule in rules2:
+            rule(state)
+        return state.tokens

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/renderer.py

@@ -0,0 +1,356 @@
+"""
+class Renderer
+
+Generates HTML from parsed token stream. Each instance has independent
+copy of rules. Those can be rewritten with ease. Also, you can add new
+rules if you create plugin and adds new token types.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+import inspect
+from typing import Any, ClassVar, Protocol
+
+from .common.utils import escapeHtml, unescapeAll
+from .token import Token
+from .utils import EnvType, OptionsDict
+
+
+class RendererProtocol(Protocol):
+    __output__: ClassVar[str]
+
+    def render(
+        self, tokens: Sequence[Token], options: OptionsDict, env: EnvType
+    ) -> Any: ...
+
+
+class RendererHTML(RendererProtocol):
+    """Contains render rules for tokens. Can be updated and extended.
+
+    Example:
+
+    Each rule is called as independent static function with fixed signature:
+
+    ::
+
+        class Renderer:
+            def token_type_name(self, tokens, idx, options, env) {
+                # ...
+                return renderedHTML
+
+    ::
+
+        class CustomRenderer(RendererHTML):
+            def strong_open(self, tokens, idx, options, env):
+                return '<b>'
+            def strong_close(self, tokens, idx, options, env):
+                return '</b>'
+
+        md = MarkdownIt(renderer_cls=CustomRenderer)
+
+        result = md.render(...)
+
+    See https://github.com/markdown-it/markdown-it/blob/master/lib/renderer.js
+    for more details and examples.
+    """
+
+    __output__ = "html"
+
+    def __init__(self, parser: Any = None):
+        self.rules = {
+            k: v
+            for k, v in inspect.getmembers(self, predicate=inspect.ismethod)
+            if not (k.startswith("render") or k.startswith("_"))
+        }
+
+    def render(
+        self, tokens: Sequence[Token], options: OptionsDict, env: EnvType
+    ) -> str:
+        """Takes token stream and generates HTML.
+
+        :param tokens: list on block tokens to render
+        :param options: params of parser instance
+        :param env: additional data from parsed input
+
+        """
+        result = ""
+
+        for i, token in enumerate(tokens):
+            if token.type == "inline":
+                if token.children:
+                    result += self.renderInline(token.children, options, env)
+            elif token.type in self.rules:
+                result += self.rules[token.type](tokens, i, options, env)
+            else:
+                result += self.renderToken(tokens, i, options, env)
+
+        return result
+
+    def renderInline(
+        self, tokens: Sequence[Token], options: OptionsDict, env: EnvType
+    ) -> str:
+        """The same as ``render``, but for single token of `inline` type.
+
+        :param tokens: list on block tokens to render
+        :param options: params of parser instance
+        :param env: additional data from parsed input (references, for example)
+        """
+        result = ""
+
+        for i, token in enumerate(tokens):
+            if token.type in self.rules:
+                result += self.rules[token.type](tokens, i, options, env)
+            else:
+                result += self.renderToken(tokens, i, options, env)
+
+        return result
+
+    def renderToken(
+        self,
+        tokens: Sequence[Token],
+        idx: int,
+        options: OptionsDict,
+        env: EnvType,
+    ) -> str:
+        """Default token renderer.
+
+        Can be overridden by custom function
+
+        :param idx: token index to render
+        :param options: params of parser instance
+        """
+        result = ""
+        needLf = False
+        token = tokens[idx]
+
+        # Tight list paragraphs
+        if token.hidden:
+            return ""
+
+        # Insert a newline between hidden paragraph and subsequent opening
+        # block-level tag.
+        #
+        # For example, here we should insert a newline before blockquote:
+        #  - a
+        #    >
+        #
+        if token.block and token.nesting != -1 and idx and tokens[idx - 1].hidden:
+            result += "\n"
+
+        # Add token name, e.g. `<img`
+        result += ("</" if token.nesting == -1 else "<") + token.tag
+
+        # Encode attributes, e.g. `<img src="foo"`
+        result += self.renderAttrs(token)
+
+        # Add a slash for self-closing tags, e.g. `<img src="foo" /`
+        if token.nesting == 0 and options["xhtmlOut"]:
+            result += " /"
+
+        # Check if we need to add a newline after this tag
+        if token.block:
+            needLf = True
+
+            if token.nesting == 1 and (idx + 1 < len(tokens)):
+                nextToken = tokens[idx + 1]
+
+                if nextToken.type == "inline" or nextToken.hidden:
+                    # Block-level tag containing an inline tag.
+                    #
+                    needLf = False
+
+                elif nextToken.nesting == -1 and nextToken.tag == token.tag:
+                    # Opening tag + closing tag of the same type. E.g. `<li></li>`.
+                    #
+                    needLf = False
+
+        result += ">\n" if needLf else ">"
+
+        return result
+
+    @staticmethod
+    def renderAttrs(token: Token) -> str:
+        """Render token attributes to string."""
+        result = ""
+
+        for key, value in token.attrItems():
+            result += " " + escapeHtml(key) + '="' + escapeHtml(str(value)) + '"'
+
+        return result
+
+    def renderInlineAsText(
+        self,
+        tokens: Sequence[Token] | None,
+        options: OptionsDict,
+        env: EnvType,
+    ) -> str:
+        """Special kludge for image `alt` attributes to conform CommonMark spec.
+
+        Don't try to use it! Spec requires to show `alt` content with stripped markup,
+        instead of simple escaping.
+
+        :param tokens: list on block tokens to render
+        :param options: params of parser instance
+        :param env: additional data from parsed input
+        """
+        result = ""
+
+        for token in tokens or []:
+            if token.type == "text":
+                result += token.content
+            elif token.type == "image":
+                if token.children:
+                    result += self.renderInlineAsText(token.children, options, env)
+            elif token.type == "softbreak":
+                result += "\n"
+
+        return result
+
+    ###################################################
+
+    def list_item_open(
+        self,
+        tokens: Sequence[Token],
+        idx: int,
+        options: OptionsDict,
+        env: EnvType,
+    ) -> str:
+        token = tokens[idx]
+        result = self.renderToken(tokens, idx, options, env)
+        if token.meta and "checked" in token.meta:
+            checked_attr = ' checked=""' if token.meta["checked"] else ""
+            disabled_attr = (
+                "" if options.get("tasklists_editable", False) else ' disabled=""'
+            )
+            result += (
+                '<input class="task-list-item-checkbox"'
+                f'{disabled_attr} type="checkbox"{checked_attr}> '
+            )
+        return result
+
+    def code_inline(
+        self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
+    ) -> str:
+        token = tokens[idx]
+        return (
+            "<code"
+            + self.renderAttrs(token)
+            + ">"
+            + escapeHtml(tokens[idx].content)
+            + "</code>"
+        )
+
+    def code_block(
+        self,
+        tokens: Sequence[Token],
+        idx: int,
+        options: OptionsDict,
+        env: EnvType,
+    ) -> str:
+        token = tokens[idx]
+
+        return (
+            "<pre"
+            + self.renderAttrs(token)
+            + "><code>"
+            + escapeHtml(tokens[idx].content)
+            + "</code></pre>\n"
+        )
+
+    def fence(
+        self,
+        tokens: Sequence[Token],
+        idx: int,
+        options: OptionsDict,
+        env: EnvType,
+    ) -> str:
+        token = tokens[idx]
+        info = unescapeAll(token.info).strip() if token.info else ""
+        langName = ""
+        langAttrs = ""
+
+        if info:
+            arr = info.split(maxsplit=1)
+            langName = arr[0]
+            if len(arr) == 2:
+                langAttrs = arr[1]
+
+        if options.highlight:
+            highlighted = options.highlight(
+                token.content, langName, langAttrs
+            ) or escapeHtml(token.content)
+        else:
+            highlighted = escapeHtml(token.content)
+
+        if highlighted.startswith("<pre"):
+            return highlighted + "\n"
+
+        # If language exists, inject class gently, without modifying original token.
+        # May be, one day we will add .deepClone() for token and simplify this part, but
+        # now we prefer to keep things local.
+        if info:
+            # Fake token just to render attributes
+            tmpToken = Token(type="", tag="", nesting=0, attrs=token.attrs.copy())
+            tmpToken.attrJoin("class", options.langPrefix + langName)
+
+            return (
+                "<pre><code"
+                + self.renderAttrs(tmpToken)
+                + ">"
+                + highlighted
+                + "</code></pre>\n"
+            )
+
+        return (
+            "<pre><code"
+            + self.renderAttrs(token)
+            + ">"
+            + highlighted
+            + "</code></pre>\n"
+        )
+
+    def image(
+        self,
+        tokens: Sequence[Token],
+        idx: int,
+        options: OptionsDict,
+        env: EnvType,
+    ) -> str:
+        token = tokens[idx]
+
+        # "alt" attr MUST be set, even if empty. Because it's mandatory and
+        # should be placed on proper position for tests.
+        if token.children:
+            token.attrSet("alt", self.renderInlineAsText(token.children, options, env))
+        else:
+            token.attrSet("alt", "")
+
+        return self.renderToken(tokens, idx, options, env)
+
+    def hardbreak(
+        self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
+    ) -> str:
+        return "<br />\n" if options.xhtmlOut else "<br>\n"
+
+    def softbreak(
+        self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
+    ) -> str:
+        return (
+            ("<br />\n" if options.xhtmlOut else "<br>\n") if options.breaks else "\n"
+        )
+
+    def text(
+        self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
+    ) -> str:
+        return escapeHtml(tokens[idx].content)
+
+    def html_block(
+        self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
+    ) -> str:
+        return tokens[idx].content
+
+    def html_inline(
+        self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
+    ) -> str:
+        return tokens[idx].content

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/ruler.py

@@ -0,0 +1,275 @@
+"""
+class Ruler
+
+Helper class, used by [[MarkdownIt#core]], [[MarkdownIt#block]] and
+[[MarkdownIt#inline]] to manage sequences of functions (rules):
+
+- keep rules in defined order
+- assign the name to each rule
+- enable/disable rules
+- add/replace rules
+- allow assign rules to additional named chains (in the same)
+- caching lists of active rules
+
+You will not need use this class directly until write plugins. For simple
+rules control use [[MarkdownIt.disable]], [[MarkdownIt.enable]] and
+[[MarkdownIt.use]].
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Generic, TypedDict, TypeVar
+import warnings
+
+from .utils import EnvType
+
+if TYPE_CHECKING:
+    from markdown_it import MarkdownIt
+
+
+class StateBase:
+    def __init__(self, src: str, md: MarkdownIt, env: EnvType):
+        self.src = src
+        self.env = env
+        self.md = md
+
+    @property
+    def src(self) -> str:
+        return self._src
+
+    @src.setter
+    def src(self, value: str) -> None:
+        self._src = value
+        self._srcCharCode: tuple[int, ...] | None = None
+
+    @property
+    def srcCharCode(self) -> tuple[int, ...]:
+        warnings.warn(
+            "StateBase.srcCharCode is deprecated. Use StateBase.src instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if self._srcCharCode is None:
+            self._srcCharCode = tuple(ord(c) for c in self._src)
+        return self._srcCharCode
+
+
+class RuleOptionsType(TypedDict, total=False):
+    alt: list[str]
+
+
+RuleFuncTv = TypeVar("RuleFuncTv")
+"""A rule function, whose signature is dependent on the state type."""
+
+
+@dataclass(slots=True)
+class Rule(Generic[RuleFuncTv]):
+    name: str
+    enabled: bool
+    fn: RuleFuncTv = field(repr=False)
+    alt: list[str]
+
+
+class Ruler(Generic[RuleFuncTv]):
+    def __init__(self) -> None:
+        # List of added rules.
+        self.__rules__: list[Rule[RuleFuncTv]] = []
+        # Cached rule chains.
+        # First level - chain name, '' for default.
+        # Second level - diginal anchor for fast filtering by charcodes.
+        self.__cache__: dict[str, list[RuleFuncTv]] | None = None
+
+    def __find__(self, name: str) -> int:
+        """Find rule index by name"""
+        for i, rule in enumerate(self.__rules__):
+            if rule.name == name:
+                return i
+        return -1
+
+    def __compile__(self) -> None:
+        """Build rules lookup cache"""
+        chains = {""}
+        # collect unique names
+        for rule in self.__rules__:
+            if not rule.enabled:
+                continue
+            for name in rule.alt:
+                chains.add(name)
+        self.__cache__ = {}
+        for chain in chains:
+            self.__cache__[chain] = []
+            for rule in self.__rules__:
+                if not rule.enabled:
+                    continue
+                if chain and (chain not in rule.alt):
+                    continue
+                self.__cache__[chain].append(rule.fn)
+
+    def at(
+        self, ruleName: str, fn: RuleFuncTv, options: RuleOptionsType | None = None
+    ) -> None:
+        """Replace rule by name with new function & options.
+
+        :param ruleName: rule name to replace.
+        :param fn: new rule function.
+        :param options: new rule options (not mandatory).
+        :raises: KeyError if name not found
+        """
+        index = self.__find__(ruleName)
+        options = options or {}
+        if index == -1:
+            raise KeyError(f"Parser rule not found: {ruleName}")
+        self.__rules__[index].fn = fn
+        self.__rules__[index].alt = options.get("alt", [])
+        self.__cache__ = None
+
+    def before(
+        self,
+        beforeName: str,
+        ruleName: str,
+        fn: RuleFuncTv,
+        options: RuleOptionsType | None = None,
+    ) -> None:
+        """Add new rule to chain before one with given name.
+
+        :param beforeName: new rule will be added before this one.
+        :param ruleName: new rule will be added before this one.
+        :param fn: new rule function.
+        :param options: new rule options (not mandatory).
+        :raises: KeyError if name not found
+        """
+        index = self.__find__(beforeName)
+        options = options or {}
+        if index == -1:
+            raise KeyError(f"Parser rule not found: {beforeName}")
+        self.__rules__.insert(
+            index, Rule[RuleFuncTv](ruleName, True, fn, options.get("alt", []))
+        )
+        self.__cache__ = None
+
+    def after(
+        self,
+        afterName: str,
+        ruleName: str,
+        fn: RuleFuncTv,
+        options: RuleOptionsType | None = None,
+    ) -> None:
+        """Add new rule to chain after one with given name.
+
+        :param afterName: new rule will be added after this one.
+        :param ruleName: new rule will be added after this one.
+        :param fn: new rule function.
+        :param options: new rule options (not mandatory).
+        :raises: KeyError if name not found
+        """
+        index = self.__find__(afterName)
+        options = options or {}
+        if index == -1:
+            raise KeyError(f"Parser rule not found: {afterName}")
+        self.__rules__.insert(
+            index + 1, Rule[RuleFuncTv](ruleName, True, fn, options.get("alt", []))
+        )
+        self.__cache__ = None
+
+    def push(
+        self, ruleName: str, fn: RuleFuncTv, options: RuleOptionsType | None = None
+    ) -> None:
+        """Push new rule to the end of chain.
+
+        :param ruleName: new rule will be added to the end of chain.
+        :param fn: new rule function.
+        :param options: new rule options (not mandatory).
+
+        """
+        self.__rules__.append(
+            Rule[RuleFuncTv](ruleName, True, fn, (options or {}).get("alt", []))
+        )
+        self.__cache__ = None
+
+    def enable(
+        self, names: str | Iterable[str], ignoreInvalid: bool = False
+    ) -> list[str]:
+        """Enable rules with given names.
+
+        :param names: name or list of rule names to enable.
+        :param ignoreInvalid: ignore errors when rule not found
+        :raises: KeyError if name not found and not ignoreInvalid
+        :return: list of found rule names
+        """
+        if isinstance(names, str):
+            names = [names]
+        result: list[str] = []
+        for name in names:
+            idx = self.__find__(name)
+            if (idx < 0) and ignoreInvalid:
+                continue
+            if (idx < 0) and not ignoreInvalid:
+                raise KeyError(f"Rules manager: invalid rule name {name}")
+            self.__rules__[idx].enabled = True
+            result.append(name)
+        self.__cache__ = None
+        return result
+
+    def enableOnly(
+        self, names: str | Iterable[str], ignoreInvalid: bool = False
+    ) -> list[str]:
+        """Enable rules with given names, and disable everything else.
+
+        :param names: name or list of rule names to enable.
+        :param ignoreInvalid: ignore errors when rule not found
+        :raises: KeyError if name not found and not ignoreInvalid
+        :return: list of found rule names
+        """
+        if isinstance(names, str):
+            names = [names]
+        for rule in self.__rules__:
+            rule.enabled = False
+        return self.enable(names, ignoreInvalid)
+
+    def disable(
+        self, names: str | Iterable[str], ignoreInvalid: bool = False
+    ) -> list[str]:
+        """Disable rules with given names.
+
+        :param names: name or list of rule names to enable.
+        :param ignoreInvalid: ignore errors when rule not found
+        :raises: KeyError if name not found and not ignoreInvalid
+        :return: list of found rule names
+        """
+        if isinstance(names, str):
+            names = [names]
+        result = []
+        for name in names:
+            idx = self.__find__(name)
+            if (idx < 0) and ignoreInvalid:
+                continue
+            if (idx < 0) and not ignoreInvalid:
+                raise KeyError(f"Rules manager: invalid rule name {name}")
+            self.__rules__[idx].enabled = False
+            result.append(name)
+        self.__cache__ = None
+        return result
+
+    def getRules(self, chainName: str = "") -> list[RuleFuncTv]:
+        """Return array of active functions (rules) for given chain name.
+        It analyzes rules configuration, compiles caches if not exists and returns result.
+
+        Default chain name is `''` (empty string). It can't be skipped.
+        That's done intentionally, to keep signature monomorphic for high speed.
+
+        """
+        if self.__cache__ is None:
+            self.__compile__()
+            assert self.__cache__ is not None
+        # Chain can be empty, if rules disabled. But we still have to return Array.
+        return self.__cache__.get(chainName, []) or []
+
+    def get_all_rules(self) -> list[str]:
+        """Return all available rule names."""
+        return [r.name for r in self.__rules__]
+
+    def get_active_rules(self) -> list[str]:
+        """Return the active rule names."""
+        return [r.name for r in self.__rules__ if r.enabled]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/tree.py

@@ -0,0 +1,333 @@
+"""A tree representation of a linear markdown-it token stream.
+
+This module is not part of upstream JavaScript markdown-it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Generator, Sequence
+import textwrap
+from typing import Any, NamedTuple, TypeVar, overload
+
+from .token import Token
+
+
+class _NesterTokens(NamedTuple):
+    opening: Token
+    closing: Token
+
+
+_NodeType = TypeVar("_NodeType", bound="SyntaxTreeNode")
+
+
+class SyntaxTreeNode:
+    """A Markdown syntax tree node.
+
+    A class that can be used to construct a tree representation of a linear
+    `markdown-it-py` token stream.
+
+    Each node in the tree represents either:
+      - root of the Markdown document
+      - a single unnested `Token`
+      - a `Token` "_open" and "_close" token pair, and the tokens nested in
+          between
+    """
+
+    def __init__(
+        self, tokens: Sequence[Token] = (), *, create_root: bool = True
+    ) -> None:
+        """Initialize a `SyntaxTreeNode` from a token stream.
+
+        If `create_root` is True, create a root node for the document.
+        """
+        # Only nodes representing an unnested token have self.token
+        self.token: Token | None = None
+
+        # Only containers have nester tokens
+        self.nester_tokens: _NesterTokens | None = None
+
+        # Root node does not have self.parent
+        self._parent: Any = None
+
+        # Empty list unless a non-empty container, or unnested token that has
+        # children (i.e. inline or img)
+        self._children: list[Any] = []
+
+        if create_root:
+            self._set_children_from_tokens(tokens)
+            return
+
+        if not tokens:
+            raise ValueError(
+                "Can only create root from empty token sequence."
+                " Set `create_root=True`."
+            )
+        elif len(tokens) == 1:
+            inline_token = tokens[0]
+            if inline_token.nesting:
+                raise ValueError(
+                    "Unequal nesting level at the start and end of token stream."
+                )
+            self.token = inline_token
+            if inline_token.children:
+                self._set_children_from_tokens(inline_token.children)
+        else:
+            self.nester_tokens = _NesterTokens(tokens[0], tokens[-1])
+            self._set_children_from_tokens(tokens[1:-1])
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.type})"
+
+    @overload
+    def __getitem__(self: _NodeType, item: int) -> _NodeType: ...
+
+    @overload
+    def __getitem__(self: _NodeType, item: slice) -> list[_NodeType]: ...
+
+    def __getitem__(self: _NodeType, item: int | slice) -> _NodeType | list[_NodeType]:
+        return self.children[item]
+
+    def to_tokens(self: _NodeType) -> list[Token]:
+        """Recover the linear token stream."""
+
+        def recursive_collect_tokens(node: _NodeType, token_list: list[Token]) -> None:
+            if node.type == "root":
+                for child in node.children:
+                    recursive_collect_tokens(child, token_list)
+            elif node.token:
+                token_list.append(node.token)
+            else:
+                assert node.nester_tokens
+                token_list.append(node.nester_tokens.opening)
+                for child in node.children:
+                    recursive_collect_tokens(child, token_list)
+                token_list.append(node.nester_tokens.closing)
+
+        tokens: list[Token] = []
+        recursive_collect_tokens(self, tokens)
+        return tokens
+
+    @property
+    def children(self: _NodeType) -> list[_NodeType]:
+        return self._children
+
+    @children.setter
+    def children(self: _NodeType, value: list[_NodeType]) -> None:
+        self._children = value
+
+    @property
+    def parent(self: _NodeType) -> _NodeType | None:
+        return self._parent  # type: ignore
+
+    @parent.setter
+    def parent(self: _NodeType, value: _NodeType | None) -> None:
+        self._parent = value
+
+    @property
+    def is_root(self) -> bool:
+        """Is the node a special root node?"""
+        return not (self.token or self.nester_tokens)
+
+    @property
+    def is_nested(self) -> bool:
+        """Is this node nested?.
+
+        Returns `True` if the node represents a `Token` pair and tokens in the
+        sequence between them, where `Token.nesting` of the first `Token` in
+        the pair is 1 and nesting of the other `Token` is -1.
+        """
+        return bool(self.nester_tokens)
+
+    @property
+    def siblings(self: _NodeType) -> Sequence[_NodeType]:
+        """Get siblings of the node.
+
+        Gets the whole group of siblings, including self.
+        """
+        if not self.parent:
+            return [self]
+        return self.parent.children
+
+    @property
+    def type(self) -> str:
+        """Get a string type of the represented syntax.
+
+        - "root" for root nodes
+        - `Token.type` if the node represents an unnested token
+        - `Token.type` of the opening token, with "_open" suffix stripped, if
+            the node represents a nester token pair
+        """
+        if self.is_root:
+            return "root"
+        if self.token:
+            return self.token.type
+        assert self.nester_tokens
+        return self.nester_tokens.opening.type.removesuffix("_open")
+
+    @property
+    def next_sibling(self: _NodeType) -> _NodeType | None:
+        """Get the next node in the sequence of siblings.
+
+        Returns `None` if this is the last sibling.
+        """
+        self_index = self.siblings.index(self)
+        if self_index + 1 < len(self.siblings):
+            return self.siblings[self_index + 1]
+        return None
+
+    @property
+    def previous_sibling(self: _NodeType) -> _NodeType | None:
+        """Get the previous node in the sequence of siblings.
+
+        Returns `None` if this is the first sibling.
+        """
+        self_index = self.siblings.index(self)
+        if self_index - 1 >= 0:
+            return self.siblings[self_index - 1]
+        return None
+
+    def _add_child(
+        self,
+        tokens: Sequence[Token],
+    ) -> None:
+        """Make a child node for `self`."""
+        child = type(self)(tokens, create_root=False)
+        child.parent = self
+        self.children.append(child)
+
+    def _set_children_from_tokens(self, tokens: Sequence[Token]) -> None:
+        """Convert the token stream to a tree structure and set the resulting
+        nodes as children of `self`."""
+        reversed_tokens = list(reversed(tokens))
+        while reversed_tokens:
+            token = reversed_tokens.pop()
+
+            if not token.nesting:
+                self._add_child([token])
+                continue
+            if token.nesting != 1:
+                raise ValueError("Invalid token nesting")
+
+            nested_tokens = [token]
+            nesting = 1
+            while reversed_tokens and nesting:
+                token = reversed_tokens.pop()
+                nested_tokens.append(token)
+                nesting += token.nesting
+            if nesting:
+                raise ValueError(f"unclosed tokens starting {nested_tokens[0]}")
+
+            self._add_child(nested_tokens)
+
+    def pretty(
+        self, *, indent: int = 2, show_text: bool = False, _current: int = 0
+    ) -> str:
+        """Create an XML style string of the tree."""
+        prefix = " " * _current
+        text = prefix + f"<{self.type}"
+        if not self.is_root and self.attrs:
+            text += " " + " ".join(f"{k}={v!r}" for k, v in self.attrs.items())
+        text += ">"
+        if (
+            show_text
+            and not self.is_root
+            and self.type in ("text", "text_special")
+            and self.content
+        ):
+            text += "\n" + textwrap.indent(self.content, prefix + " " * indent)
+        for child in self.children:
+            text += "\n" + child.pretty(
+                indent=indent, show_text=show_text, _current=_current + indent
+            )
+        return text
+
+    def walk(
+        self: _NodeType, *, include_self: bool = True
+    ) -> Generator[_NodeType, None, None]:
+        """Recursively yield all descendant nodes in the tree starting at self.
+
+        The order mimics the order of the underlying linear token
+        stream (i.e. depth first).
+        """
+        if include_self:
+            yield self
+        for child in self.children:
+            yield from child.walk(include_self=True)
+
+    # NOTE:
+    # The values of the properties defined below directly map to properties
+    # of the underlying `Token`s. A root node does not translate to a `Token`
+    # object, so calling these property getters on a root node will raise an
+    # `AttributeError`.
+    #
+    # There is no mapping for `Token.nesting` because the `is_nested` property
+    # provides that data, and can be called on any node type, including root.
+
+    def _attribute_token(self) -> Token:
+        """Return the `Token` that is used as the data source for the
+        properties defined below."""
+        if self.token:
+            return self.token
+        if self.nester_tokens:
+            return self.nester_tokens.opening
+        raise AttributeError("Root node does not have the accessed attribute")
+
+    @property
+    def tag(self) -> str:
+        """html tag name, e.g. \"p\""""
+        return self._attribute_token().tag
+
+    @property
+    def attrs(self) -> dict[str, str | int | float]:
+        """Html attributes."""
+        return self._attribute_token().attrs
+
+    def attrGet(self, name: str) -> None | str | int | float:
+        """Get the value of attribute `name`, or null if it does not exist."""
+        return self._attribute_token().attrGet(name)
+
+    @property
+    def map(self) -> tuple[int, int] | None:
+        """Source map info. Format: `tuple[ line_begin, line_end ]`"""
+        map_ = self._attribute_token().map
+        if map_:
+            # Type ignore because `Token`s attribute types are not perfect
+            return tuple(map_)  # type: ignore
+        return None
+
+    @property
+    def level(self) -> int:
+        """nesting level, the same as `state.level`"""
+        return self._attribute_token().level
+
+    @property
+    def content(self) -> str:
+        """In a case of self-closing tag (code, html, fence, etc.), it
+        has contents of this tag."""
+        return self._attribute_token().content
+
+    @property
+    def markup(self) -> str:
+        """'*' or '_' for emphasis, fence string for fence, etc."""
+        return self._attribute_token().markup
+
+    @property
+    def info(self) -> str:
+        """fence infostring"""
+        return self._attribute_token().info
+
+    @property
+    def meta(self) -> dict[Any, Any]:
+        """A place for plugins to store an arbitrary data."""
+        return self._attribute_token().meta
+
+    @property
+    def block(self) -> bool:
+        """True for block-level tokens, false for inline tokens."""
+        return self._attribute_token().block
+
+    @property
+    def hidden(self) -> bool:
+        """If it's true, ignore this element when rendering.
+        Used for tight lists to hide paragraphs."""
+        return self._attribute_token().hidden

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/utils.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, MutableMapping
+from collections.abc import MutableMapping as MutableMappingABC
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, TypedDict, cast
+
+if TYPE_CHECKING:
+    from typing_extensions import NotRequired
+
+
+EnvType = MutableMapping[str, Any]  # note: could use TypeAlias in python 3.10
+"""Type for the environment sandbox used in parsing and rendering,
+which stores mutable variables for use by plugins and rules.
+"""
+
+
+class OptionsType(TypedDict):
+    """Options for parsing."""
+
+    maxNesting: int
+    """Internal protection, recursion limit."""
+    html: bool
+    """Enable HTML tags in source."""
+    linkify: bool
+    """Enable autoconversion of URL-like texts to links."""
+    typographer: bool
+    """Enable smartquotes and replacements."""
+    quotes: str
+    """Quote characters."""
+    xhtmlOut: bool
+    """Use '/' to close single tags (<br />)."""
+    breaks: bool
+    """Convert newlines in paragraphs into <br>."""
+    langPrefix: str
+    """CSS language prefix for fenced blocks."""
+    highlight: Callable[[str, str, str], str] | None
+    """Highlighter function: (content, lang, attrs) -> str."""
+    store_labels: NotRequired[bool]
+    """Store link label in link/image token's metadata (under Token.meta['label']).
+
+    This is a Python only option, and is intended for the use of round-trip parsing.
+    """
+    tasklists: NotRequired[bool]
+    """Enable GFM task list checkbox detection in list items."""
+    alerts: NotRequired[bool]
+    """Enable GitHub-style alert detection in blockquotes."""
+    tasklists_editable: NotRequired[bool]
+    """When True, rendered task list checkboxes are interactive (no disabled attribute)."""
+    strikethrough_single_tilde: NotRequired[bool]
+    """Allow single tilde ``~text~`` for strikethrough in addition to double."""
+
+
+class PresetType(TypedDict):
+    """Preset configuration for markdown-it."""
+
+    options: OptionsType
+    """Options for parsing."""
+    components: MutableMapping[str, MutableMapping[str, list[str]]]
+    """Components for parsing and rendering."""
+
+
+class OptionsDict(MutableMappingABC):  # type: ignore
+    """A dictionary, with attribute access to core markdownit configuration options."""
+
+    # Note: ideally we would probably just remove attribute access entirely,
+    # but we keep it for backwards compatibility.
+
+    def __init__(self, options: OptionsType) -> None:
+        self._options = cast(OptionsType, dict(options))
+
+    def __getitem__(self, key: str) -> Any:
+        return self._options[key]  # type: ignore[literal-required]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._options[key] = value  # type: ignore[literal-required]
+
+    def __delitem__(self, key: str) -> None:
+        del self._options[key]  # type: ignore
+
+    def __iter__(self) -> Iterable[str]:  # type: ignore
+        return iter(self._options)
+
+    def __len__(self) -> int:
+        return len(self._options)
+
+    def __repr__(self) -> str:
+        return repr(self._options)
+
+    def __str__(self) -> str:
+        return str(self._options)
+
+    @property
+    def maxNesting(self) -> int:
+        """Internal protection, recursion limit."""
+        return self._options["maxNesting"]
+
+    @maxNesting.setter
+    def maxNesting(self, value: int) -> None:
+        self._options["maxNesting"] = value
+
+    @property
+    def html(self) -> bool:
+        """Enable HTML tags in source."""
+        return self._options["html"]
+
+    @html.setter
+    def html(self, value: bool) -> None:
+        self._options["html"] = value
+
+    @property
+    def linkify(self) -> bool:
+        """Enable autoconversion of URL-like texts to links."""
+        return self._options["linkify"]
+
+    @linkify.setter
+    def linkify(self, value: bool) -> None:
+        self._options["linkify"] = value
+
+    @property
+    def typographer(self) -> bool:
+        """Enable smartquotes and replacements."""
+        return self._options["typographer"]
+
+    @typographer.setter
+    def typographer(self, value: bool) -> None:
+        self._options["typographer"] = value
+
+    @property
+    def quotes(self) -> str:
+        """Quote characters."""
+        return self._options["quotes"]
+
+    @quotes.setter
+    def quotes(self, value: str) -> None:
+        self._options["quotes"] = value
+
+    @property
+    def xhtmlOut(self) -> bool:
+        """Use '/' to close single tags (<br />)."""
+        return self._options["xhtmlOut"]
+
+    @xhtmlOut.setter
+    def xhtmlOut(self, value: bool) -> None:
+        self._options["xhtmlOut"] = value
+
+    @property
+    def breaks(self) -> bool:
+        """Convert newlines in paragraphs into <br>."""
+        return self._options["breaks"]
+
+    @breaks.setter
+    def breaks(self, value: bool) -> None:
+        self._options["breaks"] = value
+
+    @property
+    def langPrefix(self) -> str:
+        """CSS language prefix for fenced blocks."""
+        return self._options["langPrefix"]
+
+    @langPrefix.setter
+    def langPrefix(self, value: str) -> None:
+        self._options["langPrefix"] = value
+
+    @property
+    def highlight(self) -> Callable[[str, str, str], str] | None:
+        """Highlighter function: (content, langName, langAttrs) -> escaped HTML."""
+        return self._options["highlight"]
+
+    @highlight.setter
+    def highlight(self, value: Callable[[str, str, str], str] | None) -> None:
+        self._options["highlight"] = value
+
+
+def read_fixture_file(path: str | Path) -> list[list[Any]]:
+    text = Path(path).read_text(encoding="utf-8")
+    tests = []
+    section = 0
+    last_pos = 0
+    lines = text.splitlines(keepends=True)
+    for i in range(len(lines)):
+        if lines[i].rstrip() == ".":
+            if section == 0:
+                tests.append([i, lines[i - 1].strip()])
+                section = 1
+            elif section == 1:
+                tests[-1].append("".join(lines[last_pos + 1 : i]))
+                section = 2
+            elif section == 2:
+                tests[-1].append("".join(lines[last_pos + 1 : i]))
+                section = 0
+
+            last_pos = i
+    return tests

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/colpali/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_colpali import *
+    from .modeling_colpali import *
+    from .processing_colpali import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/colpali/modular_colpali.py

@@ -0,0 +1,296 @@
+# Copyright 2024 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Optional, Union
+
+from transformers.models.paligemma.processing_paligemma import IMAGE_TOKEN, PaliGemmaProcessor, build_string_from_input
+
+from ...feature_extraction_utils import BatchFeature
+from ...image_utils import ImageInput, make_flat_list_of_images
+from ...processing_utils import ProcessingKwargs, Unpack
+from ...tokenization_utils_base import PreTokenizedInput, TextInput
+from ...utils import is_torch_available, logging
+
+
+if is_torch_available():
+    import torch
+
+logger = logging.get_logger(__name__)
+
+
+class ColPaliProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "text_kwargs": {
+            "padding": "longest",
+        },
+        "images_kwargs": {
+            "data_format": "channels_first",
+            "do_convert_rgb": True,
+        },
+        "common_kwargs": {"return_tensors": "pt"},
+    }
+
+
+class ColPaliProcessor(PaliGemmaProcessor):
+    def __init__(
+        self,
+        image_processor=None,
+        tokenizer=None,
+        chat_template=None,
+        visual_prompt_prefix: str = "Describe the image.",
+        query_prefix: str = "Question: ",
+    ):
+        r"""
+        visual_prompt_prefix (`str`, *optional*, defaults to `"Describe the image."`):
+            A string that gets tokenized and prepended to the image tokens.
+        query_prefix (`str`, *optional*, defaults to `"Question: "`):
+            A prefix to be used for the query.
+        """
+        self.visual_prompt_prefix = visual_prompt_prefix
+        self.query_prefix = query_prefix
+        super().__init__(image_processor=image_processor, tokenizer=tokenizer, chat_template=chat_template)
+
+    @property
+    def query_augmentation_token(self) -> str:
+        """
+        Return the query augmentation token.
+
+        Query augmentation buffers are used as reasoning buffers during inference.
+        """
+        return self.tokenizer.pad_token
+
+    def __call__(
+        self,
+        images: ImageInput | None = None,
+        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,
+        **kwargs: Unpack[ColPaliProcessorKwargs],
+    ) -> BatchFeature:
+        r"""
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+        """
+        output_kwargs = self._merge_kwargs(
+            ColPaliProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        suffix = output_kwargs["text_kwargs"].pop("suffix", None)
+
+        return_token_type_ids = True
+
+        if text is None and images is None:
+            raise ValueError("Either text or images must be provided")
+        if text is not None and images is not None:
+            raise ValueError("Only one of text or images can be processed at a time")
+
+        if images is not None:
+            images = self.image_processor.fetch_images(images)
+            images = make_flat_list_of_images(images)
+            texts_doc = [self.visual_prompt_prefix] * len(images)
+            images = [self.image_processor.process_image(image) for image in images]
+
+            input_strings = [
+                build_string_from_input(
+                    prompt=prompt,
+                    bos_token=self.tokenizer.bos_token,
+                    image_seq_len=self.image_seq_length,
+                    image_token=IMAGE_TOKEN,
+                    num_images=len(image_list) if isinstance(image_list, list) else 1,
+                )
+                for prompt, image_list in zip(texts_doc, images)
+            ]
+            pixel_values = self.image_processor(images, **output_kwargs["images_kwargs"])["pixel_values"]
+
+            # max_length has to account for the image tokens
+            if output_kwargs["text_kwargs"].get("max_length", None) is not None:
+                output_kwargs["text_kwargs"]["max_length"] += self.image_seq_length
+
+            inputs = self.tokenizer(
+                input_strings,
+                return_token_type_ids=return_token_type_ids,
+                **output_kwargs["text_kwargs"],
+            )
+
+            return_data = {**inputs, "pixel_values": pixel_values}
+
+            if return_token_type_ids:
+                labels = inputs["input_ids"].masked_fill(inputs["token_type_ids"] == 0, -100)
+                return_data.update({"labels": labels})
+
+            return BatchFeature(data=return_data)
+
+        elif text is not None:
+            if isinstance(text, str):
+                text = [text]
+            elif not (isinstance(text, list) and isinstance(text[0], str)):
+                raise ValueError("Text must be a string or a list of strings")
+
+            if suffix is None:
+                suffix = self.query_augmentation_token * 10
+
+            texts_query: list[str] = []
+            for query in text:
+                query = self.tokenizer.bos_token + self.query_prefix + query + suffix + "\n"
+                texts_query.append(query)
+
+            output_kwargs["text_kwargs"]["max_length"] = output_kwargs["text_kwargs"].get("max_length", 50)
+
+            batch_query = self.tokenizer(
+                texts_query,
+                return_token_type_ids=return_token_type_ids,
+                **output_kwargs["text_kwargs"],
+            )
+
+            return batch_query
+
+    def process_images(
+        self,
+        images: ImageInput | None = None,
+        **kwargs: Unpack[ColPaliProcessorKwargs],
+    ) -> BatchFeature:
+        """
+        Prepare for the model one or several image(s). This method is a wrapper around the `__call__` method of the ColPaliProcessor's
+        [`ColPaliProcessor.__call__`].
+
+        This method forwards the `images` and `kwargs` arguments to the image processor.
+
+        Args:
+            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
+                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
+                tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a
+                number of channels, H and W are image height and width.
+            return_tensors (`str` or [`~utils.TensorType`], *optional*):
+                If set, will return tensors of a particular framework. Acceptable values are:
+
+                - `'pt'`: Return PyTorch `torch.Tensor` objects.
+                - `'np'`: Return NumPy `np.ndarray` objects.
+
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+        """
+        return self.__call__(images=images, **kwargs)
+
+    def process_queries(
+        self,
+        text: TextInput | list[TextInput],
+        **kwargs: Unpack[ColPaliProcessorKwargs],
+    ) -> BatchFeature:
+        """
+        Prepare for the model one or several texts. This method is a wrapper around the `__call__` method of the ColPaliProcessor's
+        [`ColPaliProcessor.__call__`].
+
+        This method forwards the `text` and `kwargs` arguments to the tokenizer.
+
+        Args:
+            text (`str`, `list[str]`, `list[list[str]]`):
+                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
+                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
+            return_tensors (`str` or [`~utils.TensorType`], *optional*):
+                If set, will return tensors of a particular framework. Acceptable values are:
+
+                - `'pt'`: Return PyTorch `torch.Tensor` objects.
+                - `'np'`: Return NumPy `np.ndarray` objects.
+
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+        """
+        return self.__call__(text=text, **kwargs)
+
+    def score_retrieval(
+        self,
+        query_embeddings: Union["torch.Tensor", list["torch.Tensor"]],
+        passage_embeddings: Union["torch.Tensor", list["torch.Tensor"]],
+        batch_size: int = 128,
+        output_dtype: Optional["torch.dtype"] = None,
+        output_device: Union["torch.device", str] = "cpu",
+    ) -> "torch.Tensor":
+        """
+        Compute the late-interaction/MaxSim score (ColBERT-like) for the given multi-vector
+        query embeddings (`qs`) and passage embeddings (`ps`). For ColPali, a passage is the
+        image of a document page.
+
+        Because the embedding tensors are multi-vector and can thus have different shapes, they
+        should be fed as:
+        (1) a list of tensors, where the i-th tensor is of shape (sequence_length_i, embedding_dim)
+        (2) a single tensor of shape (n_passages, max_sequence_length, embedding_dim) -> usually
+            obtained by padding the list of tensors.
+
+        Args:
+            query_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Query embeddings.
+            passage_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Passage embeddings.
+            batch_size (`int`, *optional*, defaults to 128): Batch size for computing scores.
+            output_dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): The dtype of the output tensor.
+                If `None`, the dtype of the input embeddings is used.
+            output_device (`torch.device` or `str`, *optional*, defaults to "cpu"): The device of the output tensor.
+
+        Returns:
+            `torch.Tensor`: A tensor of shape `(n_queries, n_passages)` containing the scores. The score
+            tensor is saved on the "cpu" device.
+        """
+
+        if len(query_embeddings) == 0:
+            raise ValueError("No queries provided")
+        if len(passage_embeddings) == 0:
+            raise ValueError("No passages provided")
+
+        if query_embeddings[0].device != passage_embeddings[0].device:
+            raise ValueError("Queries and passages must be on the same device")
+
+        if query_embeddings[0].dtype != passage_embeddings[0].dtype:
+            raise ValueError("Queries and passages must have the same dtype")
+
+        if output_dtype is None:
+            output_dtype = query_embeddings[0].dtype
+
+        scores: list[torch.Tensor] = []
+
+        for i in range(0, len(query_embeddings), batch_size):
+            batch_scores: list[torch.Tensor] = []
+            batch_queries = torch.nn.utils.rnn.pad_sequence(
+                query_embeddings[i : i + batch_size], batch_first=True, padding_value=0
+            )
+            for j in range(0, len(passage_embeddings), batch_size):
+                batch_passages = torch.nn.utils.rnn.pad_sequence(
+                    passage_embeddings[j : j + batch_size], batch_first=True, padding_value=0
+                )
+                batch_scores.append(
+                    torch.einsum("bnd,csd->bcns", batch_queries, batch_passages).max(dim=3)[0].sum(dim=2)
+                )
+            scores.append(torch.cat(batch_scores, dim=1).to(output_dtype).to(output_device))
+
+        return torch.cat(scores, dim=0)
+
+
+__all__ = [
+    "ColPaliProcessor",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_153000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:376ea08cb78c6b4dea29f2f53d9e3b10a1d371423435d41f7cd8c3d9f9cfaf2c
+size 927700322

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_169000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b839d0620ef827b475afe6ac6662ebb2cdcb40e2d4c6802d6b4804a6c2134835
+size 927700322

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_188000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2fd16d59df0b5ba62c3caa8616d1ae7b68184ff6811d2e12f90b0bef05e7816
+size 927700322

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_247000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fd91ac84c75cac68c682ff0df325b91300c8b01d7974af2ad4af2cd5f1563b
+size 927700322

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_272000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8ff1d2e5dfb136a4d48f240a4bfb4600ea7173c966140dd7103a84bacf75569
+size 927700322

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_287000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78056d738b731e543940b219e7a6658fab4ef2a0c92cfc22eeece99c427c6cb
+size 927700322

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_288000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b7eb15fda77f5faf8da57859d476b4f6d9782fe40903bdb45da4dd024952a90
+size 927700322

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_297000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddffefa4e58da82402f0f2a270c92cee36aebdebb680e53a17d8ef2568e6f36d
+size 927700322