Add files using upload-large-folder tool
Browse files- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/__init__.py +6 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_destination.py +83 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_label.py +44 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_title.py +75 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/__init__.py +48 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/commonmark.py +75 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/default.py +36 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/zero.py +44 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/blockquote.py +368 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/heading.py +69 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/lheading.py +86 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/reference.py +235 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/__init__.py +28 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/configuration_altclip.py +246 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/processing_altclip.py +28 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_003000.pt +3 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_179000.pt +3 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_185000.pt +3 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_192000.pt +3 -0
- LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_279000.pt +3 -0
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Functions for parsing Links"""
|
| 2 |
+
|
| 3 |
+
__all__ = ("parseLinkDestination", "parseLinkLabel", "parseLinkTitle")
|
| 4 |
+
from .parse_link_destination import parseLinkDestination
|
| 5 |
+
from .parse_link_label import parseLinkLabel
|
| 6 |
+
from .parse_link_title import parseLinkTitle
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_destination.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Parse link destination
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from ..common.utils import charCodeAt, unescapeAll
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class _Result:
|
| 9 |
+
__slots__ = ("ok", "pos", "str")
|
| 10 |
+
|
| 11 |
+
def __init__(self) -> None:
|
| 12 |
+
self.ok = False
|
| 13 |
+
self.pos = 0
|
| 14 |
+
self.str = ""
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def parseLinkDestination(string: str, pos: int, maximum: int) -> _Result:
|
| 18 |
+
start = pos
|
| 19 |
+
result = _Result()
|
| 20 |
+
|
| 21 |
+
if charCodeAt(string, pos) == 0x3C: # /* < */
|
| 22 |
+
pos += 1
|
| 23 |
+
while pos < maximum:
|
| 24 |
+
code = charCodeAt(string, pos)
|
| 25 |
+
if code == 0x0A: # /* \n */)
|
| 26 |
+
return result
|
| 27 |
+
if code == 0x3C: # / * < * /
|
| 28 |
+
return result
|
| 29 |
+
if code == 0x3E: # /* > */) {
|
| 30 |
+
result.pos = pos + 1
|
| 31 |
+
result.str = unescapeAll(string[start + 1 : pos])
|
| 32 |
+
result.ok = True
|
| 33 |
+
return result
|
| 34 |
+
|
| 35 |
+
if code == 0x5C and pos + 1 < maximum: # \
|
| 36 |
+
pos += 2
|
| 37 |
+
continue
|
| 38 |
+
|
| 39 |
+
pos += 1
|
| 40 |
+
|
| 41 |
+
# no closing '>'
|
| 42 |
+
return result
|
| 43 |
+
|
| 44 |
+
# this should be ... } else { ... branch
|
| 45 |
+
|
| 46 |
+
level = 0
|
| 47 |
+
while pos < maximum:
|
| 48 |
+
code = charCodeAt(string, pos)
|
| 49 |
+
|
| 50 |
+
if code is None or code == 0x20:
|
| 51 |
+
break
|
| 52 |
+
|
| 53 |
+
# ascii control characters
|
| 54 |
+
if code < 0x20 or code == 0x7F:
|
| 55 |
+
break
|
| 56 |
+
|
| 57 |
+
if code == 0x5C and pos + 1 < maximum:
|
| 58 |
+
if charCodeAt(string, pos + 1) == 0x20:
|
| 59 |
+
break
|
| 60 |
+
pos += 2
|
| 61 |
+
continue
|
| 62 |
+
|
| 63 |
+
if code == 0x28: # /* ( */)
|
| 64 |
+
level += 1
|
| 65 |
+
if level > 32:
|
| 66 |
+
return result
|
| 67 |
+
|
| 68 |
+
if code == 0x29: # /* ) */)
|
| 69 |
+
if level == 0:
|
| 70 |
+
break
|
| 71 |
+
level -= 1
|
| 72 |
+
|
| 73 |
+
pos += 1
|
| 74 |
+
|
| 75 |
+
if start == pos:
|
| 76 |
+
return result
|
| 77 |
+
if level != 0:
|
| 78 |
+
return result
|
| 79 |
+
|
| 80 |
+
result.str = unescapeAll(string[start:pos])
|
| 81 |
+
result.pos = pos
|
| 82 |
+
result.ok = True
|
| 83 |
+
return result
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_label.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Parse link label
|
| 3 |
+
|
| 4 |
+
this function assumes that first character ("[") already matches
|
| 5 |
+
returns the end of the label
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from markdown_it.rules_inline import StateInline
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def parseLinkLabel(state: StateInline, start: int, disableNested: bool = False) -> int:
|
| 13 |
+
labelEnd = -1
|
| 14 |
+
oldPos = state.pos
|
| 15 |
+
found = False
|
| 16 |
+
|
| 17 |
+
state.pos = start + 1
|
| 18 |
+
level = 1
|
| 19 |
+
|
| 20 |
+
while state.pos < state.posMax:
|
| 21 |
+
marker = state.src[state.pos]
|
| 22 |
+
if marker == "]":
|
| 23 |
+
level -= 1
|
| 24 |
+
if level == 0:
|
| 25 |
+
found = True
|
| 26 |
+
break
|
| 27 |
+
|
| 28 |
+
prevPos = state.pos
|
| 29 |
+
state.md.inline.skipToken(state)
|
| 30 |
+
if marker == "[":
|
| 31 |
+
if prevPos == state.pos - 1:
|
| 32 |
+
# increase level if we find text `[`,
|
| 33 |
+
# which is not a part of any token
|
| 34 |
+
level += 1
|
| 35 |
+
elif disableNested:
|
| 36 |
+
state.pos = oldPos
|
| 37 |
+
return -1
|
| 38 |
+
if found:
|
| 39 |
+
labelEnd = state.pos
|
| 40 |
+
|
| 41 |
+
# restore old state
|
| 42 |
+
state.pos = oldPos
|
| 43 |
+
|
| 44 |
+
return labelEnd
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_title.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Parse link title"""
|
| 2 |
+
|
| 3 |
+
from ..common.utils import charCodeAt, unescapeAll
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class _State:
|
| 7 |
+
__slots__ = ("can_continue", "marker", "ok", "pos", "str")
|
| 8 |
+
|
| 9 |
+
def __init__(self) -> None:
|
| 10 |
+
self.ok = False
|
| 11 |
+
"""if `true`, this is a valid link title"""
|
| 12 |
+
self.can_continue = False
|
| 13 |
+
"""if `true`, this link can be continued on the next line"""
|
| 14 |
+
self.pos = 0
|
| 15 |
+
"""if `ok`, it's the position of the first character after the closing marker"""
|
| 16 |
+
self.str = ""
|
| 17 |
+
"""if `ok`, it's the unescaped title"""
|
| 18 |
+
self.marker = 0
|
| 19 |
+
"""expected closing marker character code"""
|
| 20 |
+
|
| 21 |
+
def __str__(self) -> str:
|
| 22 |
+
return self.str
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def parseLinkTitle(
|
| 26 |
+
string: str, start: int, maximum: int, prev_state: _State | None = None
|
| 27 |
+
) -> _State:
|
| 28 |
+
"""Parse link title within `str` in [start, max] range,
|
| 29 |
+
or continue previous parsing if `prev_state` is defined (equal to result of last execution).
|
| 30 |
+
"""
|
| 31 |
+
pos = start
|
| 32 |
+
state = _State()
|
| 33 |
+
|
| 34 |
+
if prev_state is not None:
|
| 35 |
+
# this is a continuation of a previous parseLinkTitle call on the next line,
|
| 36 |
+
# used in reference links only
|
| 37 |
+
state.str = prev_state.str
|
| 38 |
+
state.marker = prev_state.marker
|
| 39 |
+
else:
|
| 40 |
+
if pos >= maximum:
|
| 41 |
+
return state
|
| 42 |
+
|
| 43 |
+
marker = charCodeAt(string, pos)
|
| 44 |
+
|
| 45 |
+
# /* " */ /* ' */ /* ( */
|
| 46 |
+
if marker != 0x22 and marker != 0x27 and marker != 0x28:
|
| 47 |
+
return state
|
| 48 |
+
|
| 49 |
+
start += 1
|
| 50 |
+
pos += 1
|
| 51 |
+
|
| 52 |
+
# if opening marker is "(", switch it to closing marker ")"
|
| 53 |
+
if marker == 0x28:
|
| 54 |
+
marker = 0x29
|
| 55 |
+
|
| 56 |
+
state.marker = marker
|
| 57 |
+
|
| 58 |
+
while pos < maximum:
|
| 59 |
+
code = charCodeAt(string, pos)
|
| 60 |
+
if code == state.marker:
|
| 61 |
+
state.pos = pos + 1
|
| 62 |
+
state.str += unescapeAll(string[start:pos])
|
| 63 |
+
state.ok = True
|
| 64 |
+
return state
|
| 65 |
+
elif code == 0x28 and state.marker == 0x29: # /* ( */ /* ) */
|
| 66 |
+
return state
|
| 67 |
+
elif code == 0x5C and pos + 1 < maximum: # /* \ */
|
| 68 |
+
pos += 1
|
| 69 |
+
|
| 70 |
+
pos += 1
|
| 71 |
+
|
| 72 |
+
# no closing marker found, but this link title may continue on the next line (for references)
|
| 73 |
+
state.can_continue = True
|
| 74 |
+
state.str += unescapeAll(string[start:pos])
|
| 75 |
+
return state
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__all__ = ("commonmark", "default", "gfm_like", "gfm_like2", "js_default", "zero")
|
| 2 |
+
|
| 3 |
+
from ..utils import PresetType
|
| 4 |
+
from . import commonmark, default, zero
|
| 5 |
+
|
| 6 |
+
js_default = default
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class gfm_like: # noqa: N801
|
| 10 |
+
"""GitHub Flavoured Markdown (GFM) like.
|
| 11 |
+
|
| 12 |
+
This adds the linkify, table and strikethrough components to CommmonMark.
|
| 13 |
+
|
| 14 |
+
Note, it lacks task-list items and raw HTML filtering,
|
| 15 |
+
to meet the the full GFM specification
|
| 16 |
+
(see https://github.github.com/gfm/#autolinks-extension-).
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
@staticmethod
|
| 20 |
+
def make() -> PresetType:
|
| 21 |
+
config = commonmark.make()
|
| 22 |
+
config["components"]["core"]["rules"].append("linkify")
|
| 23 |
+
config["components"]["block"]["rules"].append("table")
|
| 24 |
+
config["components"]["inline"]["rules"].extend(["strikethrough", "linkify"])
|
| 25 |
+
config["components"]["inline"]["rules2"].append("strikethrough")
|
| 26 |
+
config["options"]["linkify"] = True
|
| 27 |
+
config["options"]["html"] = True
|
| 28 |
+
return config
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class gfm_like2: # noqa: N801
|
| 32 |
+
"""GitHub Flavoured Markdown (GFM) like, extended.
|
| 33 |
+
|
| 34 |
+
Builds on ``gfm-like`` and additionally enables:
|
| 35 |
+
|
| 36 |
+
- Task lists (``- [x] done``)
|
| 37 |
+
- Alerts (``> [!NOTE]``)
|
| 38 |
+
- Single-tilde strikethrough (``~text~`` in addition to ``~~text~~``)
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
@staticmethod
|
| 42 |
+
def make() -> PresetType:
|
| 43 |
+
config = gfm_like.make()
|
| 44 |
+
config["options"]["tasklists"] = True
|
| 45 |
+
config["options"]["tasklists_editable"] = False
|
| 46 |
+
config["options"]["alerts"] = True
|
| 47 |
+
config["options"]["strikethrough_single_tilde"] = True
|
| 48 |
+
return config
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/commonmark.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Commonmark default options.
|
| 2 |
+
|
| 3 |
+
This differs to presets.default,
|
| 4 |
+
primarily in that it allows HTML and does not enable components:
|
| 5 |
+
|
| 6 |
+
- block: table
|
| 7 |
+
- inline: strikethrough
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from ..utils import PresetType
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def make() -> PresetType:
|
| 14 |
+
return {
|
| 15 |
+
"options": {
|
| 16 |
+
"maxNesting": 20, # Internal protection, recursion limit
|
| 17 |
+
"html": True, # Enable HTML tags in source,
|
| 18 |
+
# this is just a shorthand for .enable(["html_inline", "html_block"])
|
| 19 |
+
# used by the linkify rule:
|
| 20 |
+
"linkify": False, # autoconvert URL-like texts to links
|
| 21 |
+
# used by the replacements and smartquotes rules
|
| 22 |
+
# Enable some language-neutral replacements + quotes beautification
|
| 23 |
+
"typographer": False,
|
| 24 |
+
# used by the smartquotes rule:
|
| 25 |
+
# Double + single quotes replacement pairs, when typographer enabled,
|
| 26 |
+
# and smartquotes on. Could be either a String or an Array.
|
| 27 |
+
#
|
| 28 |
+
# For example, you can use '«»„“' for Russian, '„“‚‘' for German,
|
| 29 |
+
# and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
|
| 30 |
+
"quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */
|
| 31 |
+
# Renderer specific; these options are used directly in the HTML renderer
|
| 32 |
+
"xhtmlOut": True, # Use '/' to close single tags (<br />)
|
| 33 |
+
"breaks": False, # Convert '\n' in paragraphs into <br>
|
| 34 |
+
"langPrefix": "language-", # CSS language prefix for fenced blocks
|
| 35 |
+
# Highlighter function. Should return escaped HTML,
|
| 36 |
+
# or '' if the source string is not changed and should be escaped externally.
|
| 37 |
+
# If result starts with <pre... internal wrapper is skipped.
|
| 38 |
+
#
|
| 39 |
+
# function (/*str, lang, attrs*/) { return ''; }
|
| 40 |
+
#
|
| 41 |
+
"highlight": None,
|
| 42 |
+
},
|
| 43 |
+
"components": {
|
| 44 |
+
"core": {"rules": ["normalize", "block", "inline", "text_join"]},
|
| 45 |
+
"block": {
|
| 46 |
+
"rules": [
|
| 47 |
+
"blockquote",
|
| 48 |
+
"code",
|
| 49 |
+
"fence",
|
| 50 |
+
"heading",
|
| 51 |
+
"hr",
|
| 52 |
+
"html_block",
|
| 53 |
+
"lheading",
|
| 54 |
+
"list",
|
| 55 |
+
"reference",
|
| 56 |
+
"paragraph",
|
| 57 |
+
]
|
| 58 |
+
},
|
| 59 |
+
"inline": {
|
| 60 |
+
"rules": [
|
| 61 |
+
"autolink",
|
| 62 |
+
"backticks",
|
| 63 |
+
"emphasis",
|
| 64 |
+
"entity",
|
| 65 |
+
"escape",
|
| 66 |
+
"html_inline",
|
| 67 |
+
"image",
|
| 68 |
+
"link",
|
| 69 |
+
"newline",
|
| 70 |
+
"text",
|
| 71 |
+
],
|
| 72 |
+
"rules2": ["balance_pairs", "emphasis", "fragments_join"],
|
| 73 |
+
},
|
| 74 |
+
},
|
| 75 |
+
}
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/default.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""markdown-it default options."""
|
| 2 |
+
|
| 3 |
+
from ..utils import PresetType
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def make() -> PresetType:
|
| 7 |
+
return {
|
| 8 |
+
"options": {
|
| 9 |
+
"maxNesting": 100, # Internal protection, recursion limit
|
| 10 |
+
"html": False, # Enable HTML tags in source
|
| 11 |
+
# this is just a shorthand for .disable(["html_inline", "html_block"])
|
| 12 |
+
# used by the linkify rule:
|
| 13 |
+
"linkify": False, # autoconvert URL-like texts to links
|
| 14 |
+
# used by the replacements and smartquotes rules:
|
| 15 |
+
# Enable some language-neutral replacements + quotes beautification
|
| 16 |
+
"typographer": False,
|
| 17 |
+
# used by the smartquotes rule:
|
| 18 |
+
# Double + single quotes replacement pairs, when typographer enabled,
|
| 19 |
+
# and smartquotes on. Could be either a String or an Array.
|
| 20 |
+
# For example, you can use '«»„“' for Russian, '„“‚‘' for German,
|
| 21 |
+
# and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
|
| 22 |
+
"quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */
|
| 23 |
+
# Renderer specific; these options are used directly in the HTML renderer
|
| 24 |
+
"xhtmlOut": False, # Use '/' to close single tags (<br />)
|
| 25 |
+
"breaks": False, # Convert '\n' in paragraphs into <br>
|
| 26 |
+
"langPrefix": "language-", # CSS language prefix for fenced blocks
|
| 27 |
+
# Highlighter function. Should return escaped HTML,
|
| 28 |
+
# or '' if the source string is not changed and should be escaped externally.
|
| 29 |
+
# If result starts with <pre... internal wrapper is skipped.
|
| 30 |
+
#
|
| 31 |
+
# function (/*str, lang, attrs*/) { return ''; }
|
| 32 |
+
#
|
| 33 |
+
"highlight": None,
|
| 34 |
+
},
|
| 35 |
+
"components": {"core": {}, "block": {}, "inline": {}},
|
| 36 |
+
}
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/zero.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
"Zero" preset, with nothing enabled. Useful for manual configuring of simple
|
| 3 |
+
modes. For example, to parse bold/italic only.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from ..utils import PresetType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def make() -> PresetType:
|
| 10 |
+
return {
|
| 11 |
+
"options": {
|
| 12 |
+
"maxNesting": 20, # Internal protection, recursion limit
|
| 13 |
+
"html": False, # Enable HTML tags in source
|
| 14 |
+
# this is just a shorthand for .disable(["html_inline", "html_block"])
|
| 15 |
+
# used by the linkify rule:
|
| 16 |
+
"linkify": False, # autoconvert URL-like texts to links
|
| 17 |
+
# used by the replacements and smartquotes rules:
|
| 18 |
+
# Enable some language-neutral replacements + quotes beautification
|
| 19 |
+
"typographer": False,
|
| 20 |
+
# used by the smartquotes rule:
|
| 21 |
+
# Double + single quotes replacement pairs, when typographer enabled,
|
| 22 |
+
# and smartquotes on. Could be either a String or an Array.
|
| 23 |
+
# For example, you can use '«»„“' for Russian, '„“‚‘' for German,
|
| 24 |
+
# and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
|
| 25 |
+
"quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */
|
| 26 |
+
# Renderer specific; these options are used directly in the HTML renderer
|
| 27 |
+
"xhtmlOut": False, # Use '/' to close single tags (<br />)
|
| 28 |
+
"breaks": False, # Convert '\n' in paragraphs into <br>
|
| 29 |
+
"langPrefix": "language-", # CSS language prefix for fenced blocks
|
| 30 |
+
# Highlighter function. Should return escaped HTML,
|
| 31 |
+
# or '' if the source string is not changed and should be escaped externally.
|
| 32 |
+
# If result starts with <pre... internal wrapper is skipped.
|
| 33 |
+
# function (/*str, lang, attrs*/) { return ''; }
|
| 34 |
+
"highlight": None,
|
| 35 |
+
},
|
| 36 |
+
"components": {
|
| 37 |
+
"core": {"rules": ["normalize", "block", "inline", "text_join"]},
|
| 38 |
+
"block": {"rules": ["paragraph"]},
|
| 39 |
+
"inline": {
|
| 40 |
+
"rules": ["text"],
|
| 41 |
+
"rules2": ["balance_pairs", "fragments_join"],
|
| 42 |
+
},
|
| 43 |
+
},
|
| 44 |
+
}
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/blockquote.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Block quotes
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import logging
|
| 5 |
+
|
| 6 |
+
from ..common.utils import isStrSpace
|
| 7 |
+
from .state_block import StateBlock
|
| 8 |
+
|
| 9 |
+
LOGGER = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
|
| 13 |
+
LOGGER.debug(
|
| 14 |
+
"entering blockquote: %s, %s, %s, %s", state, startLine, endLine, silent
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
oldLineMax = state.lineMax
|
| 18 |
+
pos = state.bMarks[startLine] + state.tShift[startLine]
|
| 19 |
+
max = state.eMarks[startLine]
|
| 20 |
+
|
| 21 |
+
if state.is_code_block(startLine):
|
| 22 |
+
return False
|
| 23 |
+
|
| 24 |
+
# check the block quote marker
|
| 25 |
+
try:
|
| 26 |
+
if state.src[pos] != ">":
|
| 27 |
+
return False
|
| 28 |
+
except IndexError:
|
| 29 |
+
return False
|
| 30 |
+
pos += 1
|
| 31 |
+
|
| 32 |
+
# we know that it's going to be a valid blockquote,
|
| 33 |
+
# so no point trying to find the end of it in silent mode
|
| 34 |
+
if silent:
|
| 35 |
+
return True
|
| 36 |
+
|
| 37 |
+
# set offset past spaces and ">"
|
| 38 |
+
initial = offset = state.sCount[startLine] + 1
|
| 39 |
+
|
| 40 |
+
try:
|
| 41 |
+
second_char: str | None = state.src[pos]
|
| 42 |
+
except IndexError:
|
| 43 |
+
second_char = None
|
| 44 |
+
|
| 45 |
+
# skip one optional space after '>'
|
| 46 |
+
if second_char == " ":
|
| 47 |
+
# ' > test '
|
| 48 |
+
# ^ -- position start of line here:
|
| 49 |
+
pos += 1
|
| 50 |
+
initial += 1
|
| 51 |
+
offset += 1
|
| 52 |
+
adjustTab = False
|
| 53 |
+
spaceAfterMarker = True
|
| 54 |
+
elif second_char == "\t":
|
| 55 |
+
spaceAfterMarker = True
|
| 56 |
+
|
| 57 |
+
if (state.bsCount[startLine] + offset) % 4 == 3:
|
| 58 |
+
# ' >\t test '
|
| 59 |
+
# ^ -- position start of line here (tab has width==1)
|
| 60 |
+
pos += 1
|
| 61 |
+
initial += 1
|
| 62 |
+
offset += 1
|
| 63 |
+
adjustTab = False
|
| 64 |
+
else:
|
| 65 |
+
# ' >\t test '
|
| 66 |
+
# ^ -- position start of line here + shift bsCount slightly
|
| 67 |
+
# to make extra space appear
|
| 68 |
+
adjustTab = True
|
| 69 |
+
|
| 70 |
+
else:
|
| 71 |
+
spaceAfterMarker = False
|
| 72 |
+
|
| 73 |
+
oldBMarks = [state.bMarks[startLine]]
|
| 74 |
+
state.bMarks[startLine] = pos
|
| 75 |
+
|
| 76 |
+
while pos < max:
|
| 77 |
+
ch = state.src[pos]
|
| 78 |
+
|
| 79 |
+
if isStrSpace(ch):
|
| 80 |
+
if ch == "\t":
|
| 81 |
+
offset += (
|
| 82 |
+
4
|
| 83 |
+
- (offset + state.bsCount[startLine] + (1 if adjustTab else 0)) % 4
|
| 84 |
+
)
|
| 85 |
+
else:
|
| 86 |
+
offset += 1
|
| 87 |
+
|
| 88 |
+
else:
|
| 89 |
+
break
|
| 90 |
+
|
| 91 |
+
pos += 1
|
| 92 |
+
|
| 93 |
+
oldBSCount = [state.bsCount[startLine]]
|
| 94 |
+
state.bsCount[startLine] = (
|
| 95 |
+
state.sCount[startLine] + 1 + (1 if spaceAfterMarker else 0)
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
lastLineEmpty = pos >= max
|
| 99 |
+
|
| 100 |
+
oldSCount = [state.sCount[startLine]]
|
| 101 |
+
state.sCount[startLine] = offset - initial
|
| 102 |
+
|
| 103 |
+
oldTShift = [state.tShift[startLine]]
|
| 104 |
+
state.tShift[startLine] = pos - state.bMarks[startLine]
|
| 105 |
+
|
| 106 |
+
terminatorRules = state.md.block.ruler.getRules("blockquote")
|
| 107 |
+
|
| 108 |
+
oldParentType = state.parentType
|
| 109 |
+
state.parentType = "blockquote"
|
| 110 |
+
|
| 111 |
+
# Search the end of the block
|
| 112 |
+
#
|
| 113 |
+
# Block ends with either:
|
| 114 |
+
# 1. an empty line outside:
|
| 115 |
+
# ```
|
| 116 |
+
# > test
|
| 117 |
+
#
|
| 118 |
+
# ```
|
| 119 |
+
# 2. an empty line inside:
|
| 120 |
+
# ```
|
| 121 |
+
# >
|
| 122 |
+
# test
|
| 123 |
+
# ```
|
| 124 |
+
# 3. another tag:
|
| 125 |
+
# ```
|
| 126 |
+
# > test
|
| 127 |
+
# - - -
|
| 128 |
+
# ```
|
| 129 |
+
|
| 130 |
+
# for (nextLine = startLine + 1; nextLine < endLine; nextLine++) {
|
| 131 |
+
nextLine = startLine + 1
|
| 132 |
+
while nextLine < endLine:
|
| 133 |
+
# check if it's outdented, i.e. it's inside list item and indented
|
| 134 |
+
# less than said list item:
|
| 135 |
+
#
|
| 136 |
+
# ```
|
| 137 |
+
# 1. anything
|
| 138 |
+
# > current blockquote
|
| 139 |
+
# 2. checking this line
|
| 140 |
+
# ```
|
| 141 |
+
isOutdented = state.sCount[nextLine] < state.blkIndent
|
| 142 |
+
|
| 143 |
+
pos = state.bMarks[nextLine] + state.tShift[nextLine]
|
| 144 |
+
max = state.eMarks[nextLine]
|
| 145 |
+
|
| 146 |
+
if pos >= max:
|
| 147 |
+
# Case 1: line is not inside the blockquote, and this line is empty.
|
| 148 |
+
break
|
| 149 |
+
|
| 150 |
+
evaluatesTrue = state.src[pos] == ">" and not isOutdented
|
| 151 |
+
pos += 1
|
| 152 |
+
if evaluatesTrue:
|
| 153 |
+
# This line is inside the blockquote.
|
| 154 |
+
|
| 155 |
+
# set offset past spaces and ">"
|
| 156 |
+
initial = offset = state.sCount[nextLine] + 1
|
| 157 |
+
|
| 158 |
+
try:
|
| 159 |
+
next_char: str | None = state.src[pos]
|
| 160 |
+
except IndexError:
|
| 161 |
+
next_char = None
|
| 162 |
+
|
| 163 |
+
# skip one optional space after '>'
|
| 164 |
+
if next_char == " ":
|
| 165 |
+
# ' > test '
|
| 166 |
+
# ^ -- position start of line here:
|
| 167 |
+
pos += 1
|
| 168 |
+
initial += 1
|
| 169 |
+
offset += 1
|
| 170 |
+
adjustTab = False
|
| 171 |
+
spaceAfterMarker = True
|
| 172 |
+
elif next_char == "\t":
|
| 173 |
+
spaceAfterMarker = True
|
| 174 |
+
|
| 175 |
+
if (state.bsCount[nextLine] + offset) % 4 == 3:
|
| 176 |
+
# ' >\t test '
|
| 177 |
+
# ^ -- position start of line here (tab has width==1)
|
| 178 |
+
pos += 1
|
| 179 |
+
initial += 1
|
| 180 |
+
offset += 1
|
| 181 |
+
adjustTab = False
|
| 182 |
+
else:
|
| 183 |
+
# ' >\t test '
|
| 184 |
+
# ^ -- position start of line here + shift bsCount slightly
|
| 185 |
+
# to make extra space appear
|
| 186 |
+
adjustTab = True
|
| 187 |
+
|
| 188 |
+
else:
|
| 189 |
+
spaceAfterMarker = False
|
| 190 |
+
|
| 191 |
+
oldBMarks.append(state.bMarks[nextLine])
|
| 192 |
+
state.bMarks[nextLine] = pos
|
| 193 |
+
|
| 194 |
+
while pos < max:
|
| 195 |
+
ch = state.src[pos]
|
| 196 |
+
|
| 197 |
+
if isStrSpace(ch):
|
| 198 |
+
if ch == "\t":
|
| 199 |
+
offset += (
|
| 200 |
+
4
|
| 201 |
+
- (
|
| 202 |
+
offset
|
| 203 |
+
+ state.bsCount[nextLine]
|
| 204 |
+
+ (1 if adjustTab else 0)
|
| 205 |
+
)
|
| 206 |
+
% 4
|
| 207 |
+
)
|
| 208 |
+
else:
|
| 209 |
+
offset += 1
|
| 210 |
+
else:
|
| 211 |
+
break
|
| 212 |
+
|
| 213 |
+
pos += 1
|
| 214 |
+
|
| 215 |
+
lastLineEmpty = pos >= max
|
| 216 |
+
|
| 217 |
+
oldBSCount.append(state.bsCount[nextLine])
|
| 218 |
+
state.bsCount[nextLine] = (
|
| 219 |
+
state.sCount[nextLine] + 1 + (1 if spaceAfterMarker else 0)
|
| 220 |
+
)
|
| 221 |
+
|
| 222 |
+
oldSCount.append(state.sCount[nextLine])
|
| 223 |
+
state.sCount[nextLine] = offset - initial
|
| 224 |
+
|
| 225 |
+
oldTShift.append(state.tShift[nextLine])
|
| 226 |
+
state.tShift[nextLine] = pos - state.bMarks[nextLine]
|
| 227 |
+
|
| 228 |
+
nextLine += 1
|
| 229 |
+
continue
|
| 230 |
+
|
| 231 |
+
# Case 2: line is not inside the blockquote, and the last line was empty.
|
| 232 |
+
if lastLineEmpty:
|
| 233 |
+
break
|
| 234 |
+
|
| 235 |
+
# Case 3: another tag found.
|
| 236 |
+
terminate = False
|
| 237 |
+
|
| 238 |
+
for terminatorRule in terminatorRules:
|
| 239 |
+
if terminatorRule(state, nextLine, endLine, True):
|
| 240 |
+
terminate = True
|
| 241 |
+
break
|
| 242 |
+
|
| 243 |
+
if terminate:
|
| 244 |
+
# Quirk to enforce "hard termination mode" for paragraphs;
|
| 245 |
+
# normally if you call `tokenize(state, startLine, nextLine)`,
|
| 246 |
+
# paragraphs will look below nextLine for paragraph continuation,
|
| 247 |
+
# but if blockquote is terminated by another tag, they shouldn't
|
| 248 |
+
state.lineMax = nextLine
|
| 249 |
+
|
| 250 |
+
if state.blkIndent != 0:
|
| 251 |
+
# state.blkIndent was non-zero, we now set it to zero,
|
| 252 |
+
# so we need to re-calculate all offsets to appear as
|
| 253 |
+
# if indent wasn't changed
|
| 254 |
+
oldBMarks.append(state.bMarks[nextLine])
|
| 255 |
+
oldBSCount.append(state.bsCount[nextLine])
|
| 256 |
+
oldTShift.append(state.tShift[nextLine])
|
| 257 |
+
oldSCount.append(state.sCount[nextLine])
|
| 258 |
+
state.sCount[nextLine] -= state.blkIndent
|
| 259 |
+
|
| 260 |
+
break
|
| 261 |
+
|
| 262 |
+
oldBMarks.append(state.bMarks[nextLine])
|
| 263 |
+
oldBSCount.append(state.bsCount[nextLine])
|
| 264 |
+
oldTShift.append(state.tShift[nextLine])
|
| 265 |
+
oldSCount.append(state.sCount[nextLine])
|
| 266 |
+
|
| 267 |
+
# A negative indentation means that this is a paragraph continuation
|
| 268 |
+
#
|
| 269 |
+
state.sCount[nextLine] = -1
|
| 270 |
+
|
| 271 |
+
nextLine += 1
|
| 272 |
+
|
| 273 |
+
oldIndent = state.blkIndent
|
| 274 |
+
state.blkIndent = 0
|
| 275 |
+
|
| 276 |
+
# Detect GitHub-style alert marker on the first content line.
|
| 277 |
+
# Note: `startLine` here refers to the first content line of the
|
| 278 |
+
# blockquote, after the `>` prefix has already been stripped by the
|
| 279 |
+
# blockquote parser above (bMarks/tShift adjusted to skip `> `).
|
| 280 |
+
alert_kind = None
|
| 281 |
+
if state.md.options.get("alerts", False) and nextLine > startLine:
|
| 282 |
+
alert_kind = _detect_alert(state, startLine)
|
| 283 |
+
|
| 284 |
+
lines = [startLine, 0]
|
| 285 |
+
|
| 286 |
+
if alert_kind is not None:
|
| 287 |
+
# Emit alert tokens instead of blockquote tokens
|
| 288 |
+
alert_lower = alert_kind.lower()
|
| 289 |
+
token = state.push("alert_open", "div", 1)
|
| 290 |
+
token.markup = ">"
|
| 291 |
+
token.attrSet("class", f"markdown-alert markdown-alert-{alert_lower}")
|
| 292 |
+
token.map = lines
|
| 293 |
+
token.info = alert_kind
|
| 294 |
+
token.meta = {"kind": alert_kind}
|
| 295 |
+
|
| 296 |
+
# Emit a title paragraph: <p class="markdown-alert-title">Kind</p>
|
| 297 |
+
token = state.push("alert_title_open", "p", 1)
|
| 298 |
+
token.attrSet("class", "markdown-alert-title")
|
| 299 |
+
title_token = state.push("inline", "", 0)
|
| 300 |
+
title_token.content = alert_kind.capitalize()
|
| 301 |
+
title_token.children = []
|
| 302 |
+
token = state.push("alert_title_close", "p", -1)
|
| 303 |
+
|
| 304 |
+
# Skip the marker line (startLine) and tokenize from startLine + 1.
|
| 305 |
+
contentStart = startLine + 1
|
| 306 |
+
if contentStart < nextLine:
|
| 307 |
+
# tokenize() updates state.line to nextLine as part of its
|
| 308 |
+
# contract, consistent with the blockquote code path below.
|
| 309 |
+
state.md.block.tokenize(state, contentStart, nextLine)
|
| 310 |
+
else:
|
| 311 |
+
state.line = nextLine
|
| 312 |
+
|
| 313 |
+
token = state.push("alert_close", "div", -1)
|
| 314 |
+
token.markup = ">"
|
| 315 |
+
else:
|
| 316 |
+
token = state.push("blockquote_open", "blockquote", 1)
|
| 317 |
+
token.markup = ">"
|
| 318 |
+
token.map = lines
|
| 319 |
+
|
| 320 |
+
state.md.block.tokenize(state, startLine, nextLine)
|
| 321 |
+
|
| 322 |
+
token = state.push("blockquote_close", "blockquote", -1)
|
| 323 |
+
token.markup = ">"
|
| 324 |
+
|
| 325 |
+
state.lineMax = oldLineMax
|
| 326 |
+
state.parentType = oldParentType
|
| 327 |
+
# Update the opening token map for both alert and blockquote containers.
|
| 328 |
+
lines[1] = state.line
|
| 329 |
+
|
| 330 |
+
# Restore original tShift; this might not be necessary since the parser
|
| 331 |
+
# has already been here, but just to make sure we can do that.
|
| 332 |
+
for i, item in enumerate(oldTShift):
|
| 333 |
+
state.bMarks[i + startLine] = oldBMarks[i]
|
| 334 |
+
state.tShift[i + startLine] = item
|
| 335 |
+
state.sCount[i + startLine] = oldSCount[i]
|
| 336 |
+
state.bsCount[i + startLine] = oldBSCount[i]
|
| 337 |
+
|
| 338 |
+
state.blkIndent = oldIndent
|
| 339 |
+
|
| 340 |
+
return True
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
_ALERT_TYPES = {"NOTE", "TIP", "IMPORTANT", "WARNING", "CAUTION"}
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
def _detect_alert(state: StateBlock, startLine: int) -> str | None:
|
| 347 |
+
"""Detect ``[!TYPE]`` on *startLine* (after ``>`` prefix has been stripped).
|
| 348 |
+
|
| 349 |
+
Returns the alert type string (e.g. ``"NOTE"``) or ``None``.
|
| 350 |
+
"""
|
| 351 |
+
pos = state.bMarks[startLine] + state.tShift[startLine]
|
| 352 |
+
maximum = state.eMarks[startLine]
|
| 353 |
+
src = state.src
|
| 354 |
+
|
| 355 |
+
# Trim trailing whitespace
|
| 356 |
+
while maximum > pos and src[maximum - 1] in (" ", "\t"):
|
| 357 |
+
maximum -= 1
|
| 358 |
+
|
| 359 |
+
if maximum - pos < 4:
|
| 360 |
+
return None
|
| 361 |
+
if src[pos] != "[" or src[pos + 1] != "!":
|
| 362 |
+
return None
|
| 363 |
+
if src[maximum - 1] != "]":
|
| 364 |
+
return None
|
| 365 |
+
type_str = src[pos + 2 : maximum - 1].upper()
|
| 366 |
+
if type_str not in _ALERT_TYPES:
|
| 367 |
+
return None
|
| 368 |
+
return type_str
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/heading.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Atex heading (#, ##, ...)"""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
|
| 7 |
+
from ..common.utils import isStrSpace
|
| 8 |
+
from .state_block import StateBlock
|
| 9 |
+
|
| 10 |
+
LOGGER = logging.getLogger(__name__)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def heading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
|
| 14 |
+
LOGGER.debug("entering heading: %s, %s, %s, %s", state, startLine, endLine, silent)
|
| 15 |
+
|
| 16 |
+
pos = state.bMarks[startLine] + state.tShift[startLine]
|
| 17 |
+
maximum = state.eMarks[startLine]
|
| 18 |
+
|
| 19 |
+
if state.is_code_block(startLine):
|
| 20 |
+
return False
|
| 21 |
+
|
| 22 |
+
ch: str | None = state.src[pos]
|
| 23 |
+
|
| 24 |
+
if ch != "#" or pos >= maximum:
|
| 25 |
+
return False
|
| 26 |
+
|
| 27 |
+
# count heading level
|
| 28 |
+
level = 1
|
| 29 |
+
pos += 1
|
| 30 |
+
try:
|
| 31 |
+
ch = state.src[pos]
|
| 32 |
+
except IndexError:
|
| 33 |
+
ch = None
|
| 34 |
+
while ch == "#" and pos < maximum and level <= 6:
|
| 35 |
+
level += 1
|
| 36 |
+
pos += 1
|
| 37 |
+
try:
|
| 38 |
+
ch = state.src[pos]
|
| 39 |
+
except IndexError:
|
| 40 |
+
ch = None
|
| 41 |
+
|
| 42 |
+
if level > 6 or (pos < maximum and not isStrSpace(ch)):
|
| 43 |
+
return False
|
| 44 |
+
|
| 45 |
+
if silent:
|
| 46 |
+
return True
|
| 47 |
+
|
| 48 |
+
# Let's cut tails like ' ### ' from the end of string
|
| 49 |
+
|
| 50 |
+
maximum = state.skipSpacesBack(maximum, pos)
|
| 51 |
+
tmp = state.skipCharsStrBack(maximum, "#", pos)
|
| 52 |
+
if tmp > pos and isStrSpace(state.src[tmp - 1]):
|
| 53 |
+
maximum = tmp
|
| 54 |
+
|
| 55 |
+
state.line = startLine + 1
|
| 56 |
+
|
| 57 |
+
token = state.push("heading_open", "h" + str(level), 1)
|
| 58 |
+
token.markup = "########"[:level]
|
| 59 |
+
token.map = [startLine, state.line]
|
| 60 |
+
|
| 61 |
+
token = state.push("inline", "", 0)
|
| 62 |
+
token.content = state.src[pos:maximum].strip()
|
| 63 |
+
token.map = [startLine, state.line]
|
| 64 |
+
token.children = []
|
| 65 |
+
|
| 66 |
+
token = state.push("heading_close", "h" + str(level), -1)
|
| 67 |
+
token.markup = "########"[:level]
|
| 68 |
+
|
| 69 |
+
return True
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/lheading.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# lheading (---, ==)
|
| 2 |
+
import logging
|
| 3 |
+
|
| 4 |
+
from .state_block import StateBlock
|
| 5 |
+
|
| 6 |
+
LOGGER = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def lheading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
|
| 10 |
+
LOGGER.debug("entering lheading: %s, %s, %s, %s", state, startLine, endLine, silent)
|
| 11 |
+
|
| 12 |
+
level = None
|
| 13 |
+
nextLine = startLine + 1
|
| 14 |
+
ruler = state.md.block.ruler
|
| 15 |
+
terminatorRules = ruler.getRules("paragraph")
|
| 16 |
+
|
| 17 |
+
if state.is_code_block(startLine):
|
| 18 |
+
return False
|
| 19 |
+
|
| 20 |
+
oldParentType = state.parentType
|
| 21 |
+
state.parentType = "paragraph" # use paragraph to match terminatorRules
|
| 22 |
+
|
| 23 |
+
# jump line-by-line until empty one or EOF
|
| 24 |
+
while nextLine < endLine and not state.isEmpty(nextLine):
|
| 25 |
+
# this would be a code block normally, but after paragraph
|
| 26 |
+
# it's considered a lazy continuation regardless of what's there
|
| 27 |
+
if state.sCount[nextLine] - state.blkIndent > 3:
|
| 28 |
+
nextLine += 1
|
| 29 |
+
continue
|
| 30 |
+
|
| 31 |
+
# Check for underline in setext header
|
| 32 |
+
if state.sCount[nextLine] >= state.blkIndent:
|
| 33 |
+
pos = state.bMarks[nextLine] + state.tShift[nextLine]
|
| 34 |
+
maximum = state.eMarks[nextLine]
|
| 35 |
+
|
| 36 |
+
if pos < maximum:
|
| 37 |
+
marker = state.src[pos]
|
| 38 |
+
|
| 39 |
+
if marker in ("-", "="):
|
| 40 |
+
pos = state.skipCharsStr(pos, marker)
|
| 41 |
+
pos = state.skipSpaces(pos)
|
| 42 |
+
|
| 43 |
+
# /* = */
|
| 44 |
+
if pos >= maximum:
|
| 45 |
+
level = 1 if marker == "=" else 2
|
| 46 |
+
break
|
| 47 |
+
|
| 48 |
+
# quirk for blockquotes, this line should already be checked by that rule
|
| 49 |
+
if state.sCount[nextLine] < 0:
|
| 50 |
+
nextLine += 1
|
| 51 |
+
continue
|
| 52 |
+
|
| 53 |
+
# Some tags can terminate paragraph without empty line.
|
| 54 |
+
terminate = False
|
| 55 |
+
for terminatorRule in terminatorRules:
|
| 56 |
+
if terminatorRule(state, nextLine, endLine, True):
|
| 57 |
+
terminate = True
|
| 58 |
+
break
|
| 59 |
+
if terminate:
|
| 60 |
+
break
|
| 61 |
+
|
| 62 |
+
nextLine += 1
|
| 63 |
+
|
| 64 |
+
if not level:
|
| 65 |
+
# Didn't find valid underline
|
| 66 |
+
return False
|
| 67 |
+
|
| 68 |
+
content = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
|
| 69 |
+
|
| 70 |
+
state.line = nextLine + 1
|
| 71 |
+
|
| 72 |
+
token = state.push("heading_open", "h" + str(level), 1)
|
| 73 |
+
token.markup = marker
|
| 74 |
+
token.map = [startLine, state.line]
|
| 75 |
+
|
| 76 |
+
token = state.push("inline", "", 0)
|
| 77 |
+
token.content = content
|
| 78 |
+
token.map = [startLine, state.line - 1]
|
| 79 |
+
token.children = []
|
| 80 |
+
|
| 81 |
+
token = state.push("heading_close", "h" + str(level), -1)
|
| 82 |
+
token.markup = marker
|
| 83 |
+
|
| 84 |
+
state.parentType = oldParentType
|
| 85 |
+
|
| 86 |
+
return True
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/reference.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
from ..common.utils import charCodeAt, isSpace, normalizeReference
|
| 4 |
+
from .state_block import StateBlock
|
| 5 |
+
|
| 6 |
+
LOGGER = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) -> bool:
|
| 10 |
+
LOGGER.debug(
|
| 11 |
+
"entering reference: %s, %s, %s, %s", state, startLine, _endLine, silent
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
pos = state.bMarks[startLine] + state.tShift[startLine]
|
| 15 |
+
maximum = state.eMarks[startLine]
|
| 16 |
+
nextLine = startLine + 1
|
| 17 |
+
|
| 18 |
+
if state.is_code_block(startLine):
|
| 19 |
+
return False
|
| 20 |
+
|
| 21 |
+
if state.src[pos] != "[":
|
| 22 |
+
return False
|
| 23 |
+
|
| 24 |
+
string = state.src[pos : maximum + 1]
|
| 25 |
+
|
| 26 |
+
# string = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
|
| 27 |
+
maximum = len(string)
|
| 28 |
+
|
| 29 |
+
labelEnd = None
|
| 30 |
+
pos = 1
|
| 31 |
+
while pos < maximum:
|
| 32 |
+
ch = charCodeAt(string, pos)
|
| 33 |
+
if ch == 0x5B: # /* [ */
|
| 34 |
+
return False
|
| 35 |
+
elif ch == 0x5D: # /* ] */
|
| 36 |
+
labelEnd = pos
|
| 37 |
+
break
|
| 38 |
+
elif ch == 0x0A: # /* \n */
|
| 39 |
+
if (lineContent := getNextLine(state, nextLine)) is not None:
|
| 40 |
+
string += lineContent
|
| 41 |
+
maximum = len(string)
|
| 42 |
+
nextLine += 1
|
| 43 |
+
elif ch == 0x5C: # /* \ */
|
| 44 |
+
pos += 1
|
| 45 |
+
if (
|
| 46 |
+
pos < maximum
|
| 47 |
+
and charCodeAt(string, pos) == 0x0A
|
| 48 |
+
and (lineContent := getNextLine(state, nextLine)) is not None
|
| 49 |
+
):
|
| 50 |
+
string += lineContent
|
| 51 |
+
maximum = len(string)
|
| 52 |
+
nextLine += 1
|
| 53 |
+
pos += 1
|
| 54 |
+
|
| 55 |
+
if (
|
| 56 |
+
labelEnd is None or labelEnd < 0 or charCodeAt(string, labelEnd + 1) != 0x3A
|
| 57 |
+
): # /* : */
|
| 58 |
+
return False
|
| 59 |
+
|
| 60 |
+
# [label]: destination 'title'
|
| 61 |
+
# ^^^ skip optional whitespace here
|
| 62 |
+
pos = labelEnd + 2
|
| 63 |
+
while pos < maximum:
|
| 64 |
+
ch = charCodeAt(string, pos)
|
| 65 |
+
if ch == 0x0A:
|
| 66 |
+
if (lineContent := getNextLine(state, nextLine)) is not None:
|
| 67 |
+
string += lineContent
|
| 68 |
+
maximum = len(string)
|
| 69 |
+
nextLine += 1
|
| 70 |
+
elif isSpace(ch):
|
| 71 |
+
pass
|
| 72 |
+
else:
|
| 73 |
+
break
|
| 74 |
+
pos += 1
|
| 75 |
+
|
| 76 |
+
# [label]: destination 'title'
|
| 77 |
+
# ^^^^^^^^^^^ parse this
|
| 78 |
+
destRes = state.md.helpers.parseLinkDestination(string, pos, maximum)
|
| 79 |
+
if not destRes.ok:
|
| 80 |
+
return False
|
| 81 |
+
|
| 82 |
+
href = state.md.normalizeLink(destRes.str)
|
| 83 |
+
if not state.md.validateLink(href):
|
| 84 |
+
return False
|
| 85 |
+
|
| 86 |
+
pos = destRes.pos
|
| 87 |
+
|
| 88 |
+
# save cursor state, we could require to rollback later
|
| 89 |
+
destEndPos = pos
|
| 90 |
+
destEndLineNo = nextLine
|
| 91 |
+
|
| 92 |
+
# [label]: destination 'title'
|
| 93 |
+
# ^^^ skipping those spaces
|
| 94 |
+
start = pos
|
| 95 |
+
while pos < maximum:
|
| 96 |
+
ch = charCodeAt(string, pos)
|
| 97 |
+
if ch == 0x0A:
|
| 98 |
+
if (lineContent := getNextLine(state, nextLine)) is not None:
|
| 99 |
+
string += lineContent
|
| 100 |
+
maximum = len(string)
|
| 101 |
+
nextLine += 1
|
| 102 |
+
elif isSpace(ch):
|
| 103 |
+
pass
|
| 104 |
+
else:
|
| 105 |
+
break
|
| 106 |
+
pos += 1
|
| 107 |
+
|
| 108 |
+
# [label]: destination 'title'
|
| 109 |
+
# ^^^^^^^ parse this
|
| 110 |
+
titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, None)
|
| 111 |
+
while titleRes.can_continue:
|
| 112 |
+
if (lineContent := getNextLine(state, nextLine)) is None:
|
| 113 |
+
break
|
| 114 |
+
string += lineContent
|
| 115 |
+
pos = maximum
|
| 116 |
+
maximum = len(string)
|
| 117 |
+
nextLine += 1
|
| 118 |
+
titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, titleRes)
|
| 119 |
+
|
| 120 |
+
if pos < maximum and start != pos and titleRes.ok:
|
| 121 |
+
title = titleRes.str
|
| 122 |
+
pos = titleRes.pos
|
| 123 |
+
else:
|
| 124 |
+
title = ""
|
| 125 |
+
pos = destEndPos
|
| 126 |
+
nextLine = destEndLineNo
|
| 127 |
+
|
| 128 |
+
# skip trailing spaces until the rest of the line
|
| 129 |
+
while pos < maximum:
|
| 130 |
+
ch = charCodeAt(string, pos)
|
| 131 |
+
if not isSpace(ch):
|
| 132 |
+
break
|
| 133 |
+
pos += 1
|
| 134 |
+
|
| 135 |
+
if pos < maximum and charCodeAt(string, pos) != 0x0A and title:
|
| 136 |
+
# garbage at the end of the line after title,
|
| 137 |
+
# but it could still be a valid reference if we roll back
|
| 138 |
+
title = ""
|
| 139 |
+
pos = destEndPos
|
| 140 |
+
nextLine = destEndLineNo
|
| 141 |
+
while pos < maximum:
|
| 142 |
+
ch = charCodeAt(string, pos)
|
| 143 |
+
if not isSpace(ch):
|
| 144 |
+
break
|
| 145 |
+
pos += 1
|
| 146 |
+
|
| 147 |
+
if pos < maximum and charCodeAt(string, pos) != 0x0A:
|
| 148 |
+
# garbage at the end of the line
|
| 149 |
+
return False
|
| 150 |
+
|
| 151 |
+
label = normalizeReference(string[1:labelEnd])
|
| 152 |
+
if not label:
|
| 153 |
+
# CommonMark 0.20 disallows empty labels
|
| 154 |
+
return False
|
| 155 |
+
|
| 156 |
+
# Reference can not terminate anything. This check is for safety only.
|
| 157 |
+
if silent:
|
| 158 |
+
return True
|
| 159 |
+
|
| 160 |
+
if "references" not in state.env:
|
| 161 |
+
state.env["references"] = {}
|
| 162 |
+
|
| 163 |
+
state.line = nextLine
|
| 164 |
+
|
| 165 |
+
# note, this is not part of markdown-it JS, but is useful for renderers
|
| 166 |
+
if state.md.options.get("inline_definitions", False):
|
| 167 |
+
token = state.push("definition", "", 0)
|
| 168 |
+
token.meta = {
|
| 169 |
+
"id": label,
|
| 170 |
+
"title": title,
|
| 171 |
+
"url": href,
|
| 172 |
+
"label": string[1:labelEnd],
|
| 173 |
+
}
|
| 174 |
+
token.map = [startLine, state.line]
|
| 175 |
+
|
| 176 |
+
if label not in state.env["references"]:
|
| 177 |
+
state.env["references"][label] = {
|
| 178 |
+
"title": title,
|
| 179 |
+
"href": href,
|
| 180 |
+
"map": [startLine, state.line],
|
| 181 |
+
}
|
| 182 |
+
else:
|
| 183 |
+
state.env.setdefault("duplicate_refs", []).append(
|
| 184 |
+
{
|
| 185 |
+
"title": title,
|
| 186 |
+
"href": href,
|
| 187 |
+
"label": label,
|
| 188 |
+
"map": [startLine, state.line],
|
| 189 |
+
}
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
return True
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def getNextLine(state: StateBlock, nextLine: int) -> None | str:
|
| 196 |
+
endLine = state.lineMax
|
| 197 |
+
|
| 198 |
+
if nextLine >= endLine or state.isEmpty(nextLine):
|
| 199 |
+
# empty line or end of input
|
| 200 |
+
return None
|
| 201 |
+
|
| 202 |
+
isContinuation = False
|
| 203 |
+
|
| 204 |
+
# this would be a code block normally, but after paragraph
|
| 205 |
+
# it's considered a lazy continuation regardless of what's there
|
| 206 |
+
if state.is_code_block(nextLine):
|
| 207 |
+
isContinuation = True
|
| 208 |
+
|
| 209 |
+
# quirk for blockquotes, this line should already be checked by that rule
|
| 210 |
+
if state.sCount[nextLine] < 0:
|
| 211 |
+
isContinuation = True
|
| 212 |
+
|
| 213 |
+
if not isContinuation:
|
| 214 |
+
terminatorRules = state.md.block.ruler.getRules("reference")
|
| 215 |
+
oldParentType = state.parentType
|
| 216 |
+
state.parentType = "reference"
|
| 217 |
+
|
| 218 |
+
# Some tags can terminate paragraph without empty line.
|
| 219 |
+
terminate = False
|
| 220 |
+
for terminatorRule in terminatorRules:
|
| 221 |
+
if terminatorRule(state, nextLine, endLine, True):
|
| 222 |
+
terminate = True
|
| 223 |
+
break
|
| 224 |
+
|
| 225 |
+
state.parentType = oldParentType
|
| 226 |
+
|
| 227 |
+
if terminate:
|
| 228 |
+
# terminated by another block
|
| 229 |
+
return None
|
| 230 |
+
|
| 231 |
+
pos = state.bMarks[nextLine] + state.tShift[nextLine]
|
| 232 |
+
maximum = state.eMarks[nextLine]
|
| 233 |
+
|
| 234 |
+
# max + 1 explicitly includes the newline
|
| 235 |
+
return state.src[pos : maximum + 1]
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
from typing import TYPE_CHECKING
|
| 15 |
+
|
| 16 |
+
from ...utils import _LazyModule
|
| 17 |
+
from ...utils.import_utils import define_import_structure
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
if TYPE_CHECKING:
|
| 21 |
+
from .configuration_altclip import *
|
| 22 |
+
from .modeling_altclip import *
|
| 23 |
+
from .processing_altclip import *
|
| 24 |
+
else:
|
| 25 |
+
import sys
|
| 26 |
+
|
| 27 |
+
_file = globals()["__file__"]
|
| 28 |
+
sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/configuration_altclip.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
| 2 |
+
# This file was automatically generated from src/transformers/models/altclip/modular_altclip.py.
|
| 3 |
+
# Do NOT edit this file manually as any edits will be overwritten by the generation of
|
| 4 |
+
# the file from the modular. If any change should be done, please apply the change to the
|
| 5 |
+
# modular_altclip.py file directly. One of our CI enforces this.
|
| 6 |
+
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
| 7 |
+
# Copyright 2022 The BAAI Teams Authors and The HuggingFace Inc. team. All rights reserved.
|
| 8 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 9 |
+
# you may not use this file except in compliance with the License.
|
| 10 |
+
# You may obtain a copy of the License at
|
| 11 |
+
#
|
| 12 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 13 |
+
#
|
| 14 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 15 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 16 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 17 |
+
# See the License for the specific language governing permissions and
|
| 18 |
+
# limitations under the License.
|
| 19 |
+
from huggingface_hub.dataclasses import strict
|
| 20 |
+
|
| 21 |
+
from ...configuration_utils import PreTrainedConfig
|
| 22 |
+
from ...utils import auto_docstring, logging
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
logger = logging.get_logger(__name__)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@auto_docstring(checkpoint="BAAI/AltCLIP")
|
| 29 |
+
@strict
|
| 30 |
+
class AltCLIPTextConfig(PreTrainedConfig):
|
| 31 |
+
r"""
|
| 32 |
+
project_dim (`int`, *optional*, defaults to 768):
|
| 33 |
+
The dimensions of the teacher model before the mapping layer.
|
| 34 |
+
|
| 35 |
+
Examples:
|
| 36 |
+
|
| 37 |
+
```python
|
| 38 |
+
>>> from transformers import AltCLIPTextModel, AltCLIPTextConfig
|
| 39 |
+
|
| 40 |
+
>>> # Initializing a AltCLIPTextConfig with BAAI/AltCLIP style configuration
|
| 41 |
+
>>> configuration = AltCLIPTextConfig()
|
| 42 |
+
|
| 43 |
+
>>> # Initializing a AltCLIPTextModel (with random weights) from the BAAI/AltCLIP style configuration
|
| 44 |
+
>>> model = AltCLIPTextModel(configuration)
|
| 45 |
+
|
| 46 |
+
>>> # Accessing the model configuration
|
| 47 |
+
>>> configuration = model.config
|
| 48 |
+
```"""
|
| 49 |
+
|
| 50 |
+
model_type = "altclip_text_model"
|
| 51 |
+
base_config_key = "text_config"
|
| 52 |
+
|
| 53 |
+
vocab_size: int = 250002
|
| 54 |
+
hidden_size: int = 1024
|
| 55 |
+
intermediate_size: int = 4096
|
| 56 |
+
num_hidden_layers: int = 24
|
| 57 |
+
num_attention_heads: int = 16
|
| 58 |
+
max_position_embeddings: int = 514
|
| 59 |
+
hidden_act: str = "gelu"
|
| 60 |
+
layer_norm_eps: float | None = 1e-5
|
| 61 |
+
initializer_range: float = 0.02
|
| 62 |
+
initializer_factor: float = 0.02
|
| 63 |
+
pad_token_id: int | None = 1
|
| 64 |
+
bos_token_id: int | None = 0
|
| 65 |
+
eos_token_id: int | None = 2
|
| 66 |
+
hidden_dropout_prob: int | float = 0.1
|
| 67 |
+
attention_probs_dropout_prob: int | float = 0
|
| 68 |
+
type_vocab_size: int = 1
|
| 69 |
+
project_dim: int = 768
|
| 70 |
+
|
| 71 |
+
def validate_architecture(self):
|
| 72 |
+
"""Part of `@strict`-powered validation. Validates the architecture of the config."""
|
| 73 |
+
if self.hidden_size % self.num_attention_heads != 0:
|
| 74 |
+
raise ValueError(
|
| 75 |
+
f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention "
|
| 76 |
+
f"heads ({self.num_attention_heads})."
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@auto_docstring(checkpoint="BAAI/AltCLIP")
|
| 81 |
+
@strict
|
| 82 |
+
class AltCLIPVisionConfig(PreTrainedConfig):
|
| 83 |
+
r"""
|
| 84 |
+
Example:
|
| 85 |
+
|
| 86 |
+
```python
|
| 87 |
+
>>> from transformers import AltCLIPVisionConfig, AltCLIPVisionModel
|
| 88 |
+
|
| 89 |
+
>>> # Initializing a AltCLIPVisionConfig with BAAI/AltCLIP style configuration
|
| 90 |
+
>>> configuration = AltCLIPVisionConfig()
|
| 91 |
+
|
| 92 |
+
>>> # Initializing a AltCLIPVisionModel (with random weights) from the BAAI/AltCLIP style configuration
|
| 93 |
+
>>> model = AltCLIPVisionModel(configuration)
|
| 94 |
+
|
| 95 |
+
>>> # Accessing the model configuration
|
| 96 |
+
>>> configuration = model.config
|
| 97 |
+
```"""
|
| 98 |
+
|
| 99 |
+
model_type = "altclip_vision_model"
|
| 100 |
+
base_config_key = "vision_config"
|
| 101 |
+
|
| 102 |
+
hidden_size: int = 768
|
| 103 |
+
intermediate_size: int = 3072
|
| 104 |
+
projection_dim: int = 512
|
| 105 |
+
num_hidden_layers: int = 12
|
| 106 |
+
num_attention_heads: int = 12
|
| 107 |
+
num_channels: int = 3
|
| 108 |
+
image_size: int | list[int] | tuple[int, int] | None = 224
|
| 109 |
+
patch_size: int | list[int] | tuple[int, int] | None = 32
|
| 110 |
+
hidden_act: str = "quick_gelu"
|
| 111 |
+
layer_norm_eps: float = 1e-5
|
| 112 |
+
attention_dropout: int | float | None = 0.0
|
| 113 |
+
initializer_range: float = 0.02
|
| 114 |
+
initializer_factor: float = 1.0
|
| 115 |
+
|
| 116 |
+
def validate_architecture(self):
|
| 117 |
+
"""Part of `@strict`-powered validation. Validates the architecture of the config."""
|
| 118 |
+
if self.hidden_size % self.num_attention_heads != 0:
|
| 119 |
+
raise ValueError(
|
| 120 |
+
f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention "
|
| 121 |
+
f"heads ({self.num_attention_heads})."
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
@auto_docstring(checkpoint="BAAI/AltCLIP")
|
| 126 |
+
@strict
|
| 127 |
+
class AltCLIPConfig(PreTrainedConfig):
|
| 128 |
+
r"""
|
| 129 |
+
Example:
|
| 130 |
+
|
| 131 |
+
```python
|
| 132 |
+
>>> from transformers import AltCLIPConfig, AltCLIPModel
|
| 133 |
+
|
| 134 |
+
>>> # Initializing a AltCLIPConfig with BAAI/AltCLIP style configuration
|
| 135 |
+
>>> configuration = AltCLIPConfig()
|
| 136 |
+
|
| 137 |
+
>>> # Initializing a AltCLIPModel (with random weights) from the BAAI/AltCLIP style configuration
|
| 138 |
+
>>> model = AltCLIPModel(configuration)
|
| 139 |
+
|
| 140 |
+
>>> # Accessing the model configuration
|
| 141 |
+
>>> configuration = model.config
|
| 142 |
+
|
| 143 |
+
>>> # We can also initialize a AltCLIPConfig from a AltCLIPTextConfig and a AltCLIPVisionConfig
|
| 144 |
+
|
| 145 |
+
>>> # Initializing a AltCLIPText and AltCLIPVision configuration
|
| 146 |
+
>>> config_text = AltCLIPTextConfig()
|
| 147 |
+
>>> config_vision = AltCLIPVisionConfig()
|
| 148 |
+
|
| 149 |
+
>>> config = AltCLIPConfig(text_config=config_text, vision_config=config_vision)
|
| 150 |
+
```"""
|
| 151 |
+
|
| 152 |
+
model_type = "altclip"
|
| 153 |
+
sub_configs = {"text_config": AltCLIPTextConfig, "vision_config": AltCLIPVisionConfig}
|
| 154 |
+
|
| 155 |
+
text_config: dict | AltCLIPTextConfig | None = None
|
| 156 |
+
vision_config: dict | AltCLIPVisionConfig | None = None
|
| 157 |
+
|
| 158 |
+
projection_dim: int = 768
|
| 159 |
+
logit_scale_init_value: float | int | None = 2.6592
|
| 160 |
+
initializer_factor: float | None = 1.0
|
| 161 |
+
|
| 162 |
+
def __post_init__(self, **kwargs):
|
| 163 |
+
if self.text_config is None:
|
| 164 |
+
text_config = {}
|
| 165 |
+
logger.info("`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values.")
|
| 166 |
+
elif isinstance(self.text_config, AltCLIPTextConfig):
|
| 167 |
+
text_config = self.text_config.to_dict()
|
| 168 |
+
else:
|
| 169 |
+
text_config = self.text_config
|
| 170 |
+
|
| 171 |
+
if self.vision_config is None:
|
| 172 |
+
vision_config = {}
|
| 173 |
+
logger.info("`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values.")
|
| 174 |
+
elif isinstance(self.vision_config, AltCLIPVisionConfig):
|
| 175 |
+
vision_config = self.vision_config.to_dict()
|
| 176 |
+
else:
|
| 177 |
+
vision_config = self.vision_config
|
| 178 |
+
|
| 179 |
+
# For backward compatibility check keyword args
|
| 180 |
+
# Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
|
| 181 |
+
# `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
|
| 182 |
+
# cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
|
| 183 |
+
text_config_dict = kwargs.pop("text_config_dict", None)
|
| 184 |
+
vision_config_dict = kwargs.pop("vision_config_dict", None)
|
| 185 |
+
|
| 186 |
+
if text_config_dict is not None:
|
| 187 |
+
# This is the complete result when using `text_config_dict`.
|
| 188 |
+
_text_config_dict = AltCLIPTextConfig(**text_config_dict).to_dict()
|
| 189 |
+
|
| 190 |
+
# Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
|
| 191 |
+
for key, value in _text_config_dict.items():
|
| 192 |
+
if key in text_config and value != text_config[key] and key != "transformers_version":
|
| 193 |
+
# If specified in `text_config_dict`
|
| 194 |
+
if key in text_config_dict:
|
| 195 |
+
message = (
|
| 196 |
+
f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
|
| 197 |
+
f'The value `text_config_dict["{key}"]` will be used instead.'
|
| 198 |
+
)
|
| 199 |
+
# If inferred from default argument values (just to be super careful)
|
| 200 |
+
else:
|
| 201 |
+
message = (
|
| 202 |
+
f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
|
| 203 |
+
f'value `text_config["{key}"]` will be overridden.'
|
| 204 |
+
)
|
| 205 |
+
logger.info(message)
|
| 206 |
+
|
| 207 |
+
# Update all values in `text_config` with the ones in `_text_config_dict`.
|
| 208 |
+
text_config.update(_text_config_dict)
|
| 209 |
+
|
| 210 |
+
if vision_config_dict is not None:
|
| 211 |
+
# This is the complete result when using `vision_config_dict`.
|
| 212 |
+
_vision_config_dict = AltCLIPVisionConfig(**vision_config_dict).to_dict()
|
| 213 |
+
# convert keys to string instead of integer
|
| 214 |
+
if "id2label" in _vision_config_dict:
|
| 215 |
+
_vision_config_dict["id2label"] = {
|
| 216 |
+
str(key): value for key, value in _vision_config_dict["id2label"].items()
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
# Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
|
| 220 |
+
for key, value in _vision_config_dict.items():
|
| 221 |
+
if key in vision_config and value != vision_config[key] and key != "transformers_version":
|
| 222 |
+
# If specified in `vision_config_dict`
|
| 223 |
+
if key in vision_config_dict:
|
| 224 |
+
message = (
|
| 225 |
+
f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
|
| 226 |
+
f'values. The value `vision_config_dict["{key}"]` will be used instead.'
|
| 227 |
+
)
|
| 228 |
+
# If inferred from default argument values (just to be super careful)
|
| 229 |
+
else:
|
| 230 |
+
message = (
|
| 231 |
+
f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
|
| 232 |
+
f'The value `vision_config["{key}"]` will be overridden.'
|
| 233 |
+
)
|
| 234 |
+
logger.info(message)
|
| 235 |
+
|
| 236 |
+
# Update all values in `vision_config` with the ones in `_vision_config_dict`.
|
| 237 |
+
vision_config.update(_vision_config_dict)
|
| 238 |
+
|
| 239 |
+
# Finally we can convert back our unified text/vision configs to `PretrainedConfig`
|
| 240 |
+
self.text_config = AltCLIPTextConfig(**text_config)
|
| 241 |
+
self.vision_config = AltCLIPVisionConfig(**vision_config)
|
| 242 |
+
|
| 243 |
+
super().__post_init__(**kwargs)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
__all__ = ["AltCLIPTextConfig", "AltCLIPVisionConfig", "AltCLIPConfig"]
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/processing_altclip.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022 WenXiang ZhongzhiCheng LedellWu LiuGuang BoWenZhang The HuggingFace Inc. team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""
|
| 15 |
+
Image/Text processor class for AltCLIP
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from ...processing_utils import ProcessorMixin
|
| 19 |
+
from ...utils import auto_docstring
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@auto_docstring
|
| 23 |
+
class AltCLIPProcessor(ProcessorMixin):
|
| 24 |
+
def __init__(self, image_processor=None, tokenizer=None):
|
| 25 |
+
super().__init__(image_processor, tokenizer)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
__all__ = ["AltCLIPProcessor"]
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_003000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aa6d3de70bd52b21c01abd539fe39af93889121f3ac7bffb10d4cafa55882001
|
| 3 |
+
size 927700322
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_179000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bbc0a9693f5ee8aec606d5b7c2dfa61a4723f697272c48d06e3393ea93687f4d
|
| 3 |
+
size 927700322
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_185000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:60f25cc52272898fec4310310b99b1c39b8a05c025b64c1cc37107ca1d354afb
|
| 3 |
+
size 927700322
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_192000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d90c3432ab622266946dd0cd272e89740e6f41214d5aab98a170b9172d88f8e2
|
| 3 |
+
size 927700322
|
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_279000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:64654fc51957676402ac4ab478eec65183ad55382beac9e45ff6df293bab67be
|
| 3 |
+
size 927700322
|