JinghuiLuAstronaut commited on
Commit
b6aa186
·
verified ·
1 Parent(s): e7c613d

Add files using upload-large-folder tool

Browse files
Files changed (20) hide show
  1. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/__init__.py +6 -0
  2. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_destination.py +83 -0
  3. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_label.py +44 -0
  4. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_title.py +75 -0
  5. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/__init__.py +48 -0
  6. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/commonmark.py +75 -0
  7. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/default.py +36 -0
  8. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/zero.py +44 -0
  9. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/blockquote.py +368 -0
  10. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/heading.py +69 -0
  11. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/lheading.py +86 -0
  12. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/reference.py +235 -0
  13. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/__init__.py +28 -0
  14. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/configuration_altclip.py +246 -0
  15. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/processing_altclip.py +28 -0
  16. LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_003000.pt +3 -0
  17. LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_179000.pt +3 -0
  18. LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_185000.pt +3 -0
  19. LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_192000.pt +3 -0
  20. LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_279000.pt +3 -0
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """Functions for parsing Links"""
2
+
3
+ __all__ = ("parseLinkDestination", "parseLinkLabel", "parseLinkTitle")
4
+ from .parse_link_destination import parseLinkDestination
5
+ from .parse_link_label import parseLinkLabel
6
+ from .parse_link_title import parseLinkTitle
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_destination.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Parse link destination
3
+ """
4
+
5
+ from ..common.utils import charCodeAt, unescapeAll
6
+
7
+
8
+ class _Result:
9
+ __slots__ = ("ok", "pos", "str")
10
+
11
+ def __init__(self) -> None:
12
+ self.ok = False
13
+ self.pos = 0
14
+ self.str = ""
15
+
16
+
17
+ def parseLinkDestination(string: str, pos: int, maximum: int) -> _Result:
18
+ start = pos
19
+ result = _Result()
20
+
21
+ if charCodeAt(string, pos) == 0x3C: # /* < */
22
+ pos += 1
23
+ while pos < maximum:
24
+ code = charCodeAt(string, pos)
25
+ if code == 0x0A: # /* \n */)
26
+ return result
27
+ if code == 0x3C: # / * < * /
28
+ return result
29
+ if code == 0x3E: # /* > */) {
30
+ result.pos = pos + 1
31
+ result.str = unescapeAll(string[start + 1 : pos])
32
+ result.ok = True
33
+ return result
34
+
35
+ if code == 0x5C and pos + 1 < maximum: # \
36
+ pos += 2
37
+ continue
38
+
39
+ pos += 1
40
+
41
+ # no closing '>'
42
+ return result
43
+
44
+ # this should be ... } else { ... branch
45
+
46
+ level = 0
47
+ while pos < maximum:
48
+ code = charCodeAt(string, pos)
49
+
50
+ if code is None or code == 0x20:
51
+ break
52
+
53
+ # ascii control characters
54
+ if code < 0x20 or code == 0x7F:
55
+ break
56
+
57
+ if code == 0x5C and pos + 1 < maximum:
58
+ if charCodeAt(string, pos + 1) == 0x20:
59
+ break
60
+ pos += 2
61
+ continue
62
+
63
+ if code == 0x28: # /* ( */)
64
+ level += 1
65
+ if level > 32:
66
+ return result
67
+
68
+ if code == 0x29: # /* ) */)
69
+ if level == 0:
70
+ break
71
+ level -= 1
72
+
73
+ pos += 1
74
+
75
+ if start == pos:
76
+ return result
77
+ if level != 0:
78
+ return result
79
+
80
+ result.str = unescapeAll(string[start:pos])
81
+ result.pos = pos
82
+ result.ok = True
83
+ return result
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_label.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Parse link label
3
+
4
+ this function assumes that first character ("[") already matches
5
+ returns the end of the label
6
+
7
+ """
8
+
9
+ from markdown_it.rules_inline import StateInline
10
+
11
+
12
+ def parseLinkLabel(state: StateInline, start: int, disableNested: bool = False) -> int:
13
+ labelEnd = -1
14
+ oldPos = state.pos
15
+ found = False
16
+
17
+ state.pos = start + 1
18
+ level = 1
19
+
20
+ while state.pos < state.posMax:
21
+ marker = state.src[state.pos]
22
+ if marker == "]":
23
+ level -= 1
24
+ if level == 0:
25
+ found = True
26
+ break
27
+
28
+ prevPos = state.pos
29
+ state.md.inline.skipToken(state)
30
+ if marker == "[":
31
+ if prevPos == state.pos - 1:
32
+ # increase level if we find text `[`,
33
+ # which is not a part of any token
34
+ level += 1
35
+ elif disableNested:
36
+ state.pos = oldPos
37
+ return -1
38
+ if found:
39
+ labelEnd = state.pos
40
+
41
+ # restore old state
42
+ state.pos = oldPos
43
+
44
+ return labelEnd
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/helpers/parse_link_title.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Parse link title"""
2
+
3
+ from ..common.utils import charCodeAt, unescapeAll
4
+
5
+
6
+ class _State:
7
+ __slots__ = ("can_continue", "marker", "ok", "pos", "str")
8
+
9
+ def __init__(self) -> None:
10
+ self.ok = False
11
+ """if `true`, this is a valid link title"""
12
+ self.can_continue = False
13
+ """if `true`, this link can be continued on the next line"""
14
+ self.pos = 0
15
+ """if `ok`, it's the position of the first character after the closing marker"""
16
+ self.str = ""
17
+ """if `ok`, it's the unescaped title"""
18
+ self.marker = 0
19
+ """expected closing marker character code"""
20
+
21
+ def __str__(self) -> str:
22
+ return self.str
23
+
24
+
25
+ def parseLinkTitle(
26
+ string: str, start: int, maximum: int, prev_state: _State | None = None
27
+ ) -> _State:
28
+ """Parse link title within `str` in [start, max] range,
29
+ or continue previous parsing if `prev_state` is defined (equal to result of last execution).
30
+ """
31
+ pos = start
32
+ state = _State()
33
+
34
+ if prev_state is not None:
35
+ # this is a continuation of a previous parseLinkTitle call on the next line,
36
+ # used in reference links only
37
+ state.str = prev_state.str
38
+ state.marker = prev_state.marker
39
+ else:
40
+ if pos >= maximum:
41
+ return state
42
+
43
+ marker = charCodeAt(string, pos)
44
+
45
+ # /* " */ /* ' */ /* ( */
46
+ if marker != 0x22 and marker != 0x27 and marker != 0x28:
47
+ return state
48
+
49
+ start += 1
50
+ pos += 1
51
+
52
+ # if opening marker is "(", switch it to closing marker ")"
53
+ if marker == 0x28:
54
+ marker = 0x29
55
+
56
+ state.marker = marker
57
+
58
+ while pos < maximum:
59
+ code = charCodeAt(string, pos)
60
+ if code == state.marker:
61
+ state.pos = pos + 1
62
+ state.str += unescapeAll(string[start:pos])
63
+ state.ok = True
64
+ return state
65
+ elif code == 0x28 and state.marker == 0x29: # /* ( */ /* ) */
66
+ return state
67
+ elif code == 0x5C and pos + 1 < maximum: # /* \ */
68
+ pos += 1
69
+
70
+ pos += 1
71
+
72
+ # no closing marker found, but this link title may continue on the next line (for references)
73
+ state.can_continue = True
74
+ state.str += unescapeAll(string[start:pos])
75
+ return state
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/__init__.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __all__ = ("commonmark", "default", "gfm_like", "gfm_like2", "js_default", "zero")
2
+
3
+ from ..utils import PresetType
4
+ from . import commonmark, default, zero
5
+
6
+ js_default = default
7
+
8
+
9
+ class gfm_like: # noqa: N801
10
+ """GitHub Flavoured Markdown (GFM) like.
11
+
12
+ This adds the linkify, table and strikethrough components to CommmonMark.
13
+
14
+ Note, it lacks task-list items and raw HTML filtering,
15
+ to meet the the full GFM specification
16
+ (see https://github.github.com/gfm/#autolinks-extension-).
17
+ """
18
+
19
+ @staticmethod
20
+ def make() -> PresetType:
21
+ config = commonmark.make()
22
+ config["components"]["core"]["rules"].append("linkify")
23
+ config["components"]["block"]["rules"].append("table")
24
+ config["components"]["inline"]["rules"].extend(["strikethrough", "linkify"])
25
+ config["components"]["inline"]["rules2"].append("strikethrough")
26
+ config["options"]["linkify"] = True
27
+ config["options"]["html"] = True
28
+ return config
29
+
30
+
31
+ class gfm_like2: # noqa: N801
32
+ """GitHub Flavoured Markdown (GFM) like, extended.
33
+
34
+ Builds on ``gfm-like`` and additionally enables:
35
+
36
+ - Task lists (``- [x] done``)
37
+ - Alerts (``> [!NOTE]``)
38
+ - Single-tilde strikethrough (``~text~`` in addition to ``~~text~~``)
39
+ """
40
+
41
+ @staticmethod
42
+ def make() -> PresetType:
43
+ config = gfm_like.make()
44
+ config["options"]["tasklists"] = True
45
+ config["options"]["tasklists_editable"] = False
46
+ config["options"]["alerts"] = True
47
+ config["options"]["strikethrough_single_tilde"] = True
48
+ return config
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/commonmark.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Commonmark default options.
2
+
3
+ This differs to presets.default,
4
+ primarily in that it allows HTML and does not enable components:
5
+
6
+ - block: table
7
+ - inline: strikethrough
8
+ """
9
+
10
+ from ..utils import PresetType
11
+
12
+
13
+ def make() -> PresetType:
14
+ return {
15
+ "options": {
16
+ "maxNesting": 20, # Internal protection, recursion limit
17
+ "html": True, # Enable HTML tags in source,
18
+ # this is just a shorthand for .enable(["html_inline", "html_block"])
19
+ # used by the linkify rule:
20
+ "linkify": False, # autoconvert URL-like texts to links
21
+ # used by the replacements and smartquotes rules
22
+ # Enable some language-neutral replacements + quotes beautification
23
+ "typographer": False,
24
+ # used by the smartquotes rule:
25
+ # Double + single quotes replacement pairs, when typographer enabled,
26
+ # and smartquotes on. Could be either a String or an Array.
27
+ #
28
+ # For example, you can use '«»„“' for Russian, '„“‚‘' for German,
29
+ # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
30
+ "quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */
31
+ # Renderer specific; these options are used directly in the HTML renderer
32
+ "xhtmlOut": True, # Use '/' to close single tags (<br />)
33
+ "breaks": False, # Convert '\n' in paragraphs into <br>
34
+ "langPrefix": "language-", # CSS language prefix for fenced blocks
35
+ # Highlighter function. Should return escaped HTML,
36
+ # or '' if the source string is not changed and should be escaped externally.
37
+ # If result starts with <pre... internal wrapper is skipped.
38
+ #
39
+ # function (/*str, lang, attrs*/) { return ''; }
40
+ #
41
+ "highlight": None,
42
+ },
43
+ "components": {
44
+ "core": {"rules": ["normalize", "block", "inline", "text_join"]},
45
+ "block": {
46
+ "rules": [
47
+ "blockquote",
48
+ "code",
49
+ "fence",
50
+ "heading",
51
+ "hr",
52
+ "html_block",
53
+ "lheading",
54
+ "list",
55
+ "reference",
56
+ "paragraph",
57
+ ]
58
+ },
59
+ "inline": {
60
+ "rules": [
61
+ "autolink",
62
+ "backticks",
63
+ "emphasis",
64
+ "entity",
65
+ "escape",
66
+ "html_inline",
67
+ "image",
68
+ "link",
69
+ "newline",
70
+ "text",
71
+ ],
72
+ "rules2": ["balance_pairs", "emphasis", "fragments_join"],
73
+ },
74
+ },
75
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/default.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """markdown-it default options."""
2
+
3
+ from ..utils import PresetType
4
+
5
+
6
+ def make() -> PresetType:
7
+ return {
8
+ "options": {
9
+ "maxNesting": 100, # Internal protection, recursion limit
10
+ "html": False, # Enable HTML tags in source
11
+ # this is just a shorthand for .disable(["html_inline", "html_block"])
12
+ # used by the linkify rule:
13
+ "linkify": False, # autoconvert URL-like texts to links
14
+ # used by the replacements and smartquotes rules:
15
+ # Enable some language-neutral replacements + quotes beautification
16
+ "typographer": False,
17
+ # used by the smartquotes rule:
18
+ # Double + single quotes replacement pairs, when typographer enabled,
19
+ # and smartquotes on. Could be either a String or an Array.
20
+ # For example, you can use '«»„“' for Russian, '„“‚‘' for German,
21
+ # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
22
+ "quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */
23
+ # Renderer specific; these options are used directly in the HTML renderer
24
+ "xhtmlOut": False, # Use '/' to close single tags (<br />)
25
+ "breaks": False, # Convert '\n' in paragraphs into <br>
26
+ "langPrefix": "language-", # CSS language prefix for fenced blocks
27
+ # Highlighter function. Should return escaped HTML,
28
+ # or '' if the source string is not changed and should be escaped externally.
29
+ # If result starts with <pre... internal wrapper is skipped.
30
+ #
31
+ # function (/*str, lang, attrs*/) { return ''; }
32
+ #
33
+ "highlight": None,
34
+ },
35
+ "components": {"core": {}, "block": {}, "inline": {}},
36
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/presets/zero.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ "Zero" preset, with nothing enabled. Useful for manual configuring of simple
3
+ modes. For example, to parse bold/italic only.
4
+ """
5
+
6
+ from ..utils import PresetType
7
+
8
+
9
+ def make() -> PresetType:
10
+ return {
11
+ "options": {
12
+ "maxNesting": 20, # Internal protection, recursion limit
13
+ "html": False, # Enable HTML tags in source
14
+ # this is just a shorthand for .disable(["html_inline", "html_block"])
15
+ # used by the linkify rule:
16
+ "linkify": False, # autoconvert URL-like texts to links
17
+ # used by the replacements and smartquotes rules:
18
+ # Enable some language-neutral replacements + quotes beautification
19
+ "typographer": False,
20
+ # used by the smartquotes rule:
21
+ # Double + single quotes replacement pairs, when typographer enabled,
22
+ # and smartquotes on. Could be either a String or an Array.
23
+ # For example, you can use '«»„“' for Russian, '„“‚‘' for German,
24
+ # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
25
+ "quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */
26
+ # Renderer specific; these options are used directly in the HTML renderer
27
+ "xhtmlOut": False, # Use '/' to close single tags (<br />)
28
+ "breaks": False, # Convert '\n' in paragraphs into <br>
29
+ "langPrefix": "language-", # CSS language prefix for fenced blocks
30
+ # Highlighter function. Should return escaped HTML,
31
+ # or '' if the source string is not changed and should be escaped externally.
32
+ # If result starts with <pre... internal wrapper is skipped.
33
+ # function (/*str, lang, attrs*/) { return ''; }
34
+ "highlight": None,
35
+ },
36
+ "components": {
37
+ "core": {"rules": ["normalize", "block", "inline", "text_join"]},
38
+ "block": {"rules": ["paragraph"]},
39
+ "inline": {
40
+ "rules": ["text"],
41
+ "rules2": ["balance_pairs", "fragments_join"],
42
+ },
43
+ },
44
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/blockquote.py ADDED
@@ -0,0 +1,368 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Block quotes
2
+ from __future__ import annotations
3
+
4
+ import logging
5
+
6
+ from ..common.utils import isStrSpace
7
+ from .state_block import StateBlock
8
+
9
+ LOGGER = logging.getLogger(__name__)
10
+
11
+
12
+ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
13
+ LOGGER.debug(
14
+ "entering blockquote: %s, %s, %s, %s", state, startLine, endLine, silent
15
+ )
16
+
17
+ oldLineMax = state.lineMax
18
+ pos = state.bMarks[startLine] + state.tShift[startLine]
19
+ max = state.eMarks[startLine]
20
+
21
+ if state.is_code_block(startLine):
22
+ return False
23
+
24
+ # check the block quote marker
25
+ try:
26
+ if state.src[pos] != ">":
27
+ return False
28
+ except IndexError:
29
+ return False
30
+ pos += 1
31
+
32
+ # we know that it's going to be a valid blockquote,
33
+ # so no point trying to find the end of it in silent mode
34
+ if silent:
35
+ return True
36
+
37
+ # set offset past spaces and ">"
38
+ initial = offset = state.sCount[startLine] + 1
39
+
40
+ try:
41
+ second_char: str | None = state.src[pos]
42
+ except IndexError:
43
+ second_char = None
44
+
45
+ # skip one optional space after '>'
46
+ if second_char == " ":
47
+ # ' > test '
48
+ # ^ -- position start of line here:
49
+ pos += 1
50
+ initial += 1
51
+ offset += 1
52
+ adjustTab = False
53
+ spaceAfterMarker = True
54
+ elif second_char == "\t":
55
+ spaceAfterMarker = True
56
+
57
+ if (state.bsCount[startLine] + offset) % 4 == 3:
58
+ # ' >\t test '
59
+ # ^ -- position start of line here (tab has width==1)
60
+ pos += 1
61
+ initial += 1
62
+ offset += 1
63
+ adjustTab = False
64
+ else:
65
+ # ' >\t test '
66
+ # ^ -- position start of line here + shift bsCount slightly
67
+ # to make extra space appear
68
+ adjustTab = True
69
+
70
+ else:
71
+ spaceAfterMarker = False
72
+
73
+ oldBMarks = [state.bMarks[startLine]]
74
+ state.bMarks[startLine] = pos
75
+
76
+ while pos < max:
77
+ ch = state.src[pos]
78
+
79
+ if isStrSpace(ch):
80
+ if ch == "\t":
81
+ offset += (
82
+ 4
83
+ - (offset + state.bsCount[startLine] + (1 if adjustTab else 0)) % 4
84
+ )
85
+ else:
86
+ offset += 1
87
+
88
+ else:
89
+ break
90
+
91
+ pos += 1
92
+
93
+ oldBSCount = [state.bsCount[startLine]]
94
+ state.bsCount[startLine] = (
95
+ state.sCount[startLine] + 1 + (1 if spaceAfterMarker else 0)
96
+ )
97
+
98
+ lastLineEmpty = pos >= max
99
+
100
+ oldSCount = [state.sCount[startLine]]
101
+ state.sCount[startLine] = offset - initial
102
+
103
+ oldTShift = [state.tShift[startLine]]
104
+ state.tShift[startLine] = pos - state.bMarks[startLine]
105
+
106
+ terminatorRules = state.md.block.ruler.getRules("blockquote")
107
+
108
+ oldParentType = state.parentType
109
+ state.parentType = "blockquote"
110
+
111
+ # Search the end of the block
112
+ #
113
+ # Block ends with either:
114
+ # 1. an empty line outside:
115
+ # ```
116
+ # > test
117
+ #
118
+ # ```
119
+ # 2. an empty line inside:
120
+ # ```
121
+ # >
122
+ # test
123
+ # ```
124
+ # 3. another tag:
125
+ # ```
126
+ # > test
127
+ # - - -
128
+ # ```
129
+
130
+ # for (nextLine = startLine + 1; nextLine < endLine; nextLine++) {
131
+ nextLine = startLine + 1
132
+ while nextLine < endLine:
133
+ # check if it's outdented, i.e. it's inside list item and indented
134
+ # less than said list item:
135
+ #
136
+ # ```
137
+ # 1. anything
138
+ # > current blockquote
139
+ # 2. checking this line
140
+ # ```
141
+ isOutdented = state.sCount[nextLine] < state.blkIndent
142
+
143
+ pos = state.bMarks[nextLine] + state.tShift[nextLine]
144
+ max = state.eMarks[nextLine]
145
+
146
+ if pos >= max:
147
+ # Case 1: line is not inside the blockquote, and this line is empty.
148
+ break
149
+
150
+ evaluatesTrue = state.src[pos] == ">" and not isOutdented
151
+ pos += 1
152
+ if evaluatesTrue:
153
+ # This line is inside the blockquote.
154
+
155
+ # set offset past spaces and ">"
156
+ initial = offset = state.sCount[nextLine] + 1
157
+
158
+ try:
159
+ next_char: str | None = state.src[pos]
160
+ except IndexError:
161
+ next_char = None
162
+
163
+ # skip one optional space after '>'
164
+ if next_char == " ":
165
+ # ' > test '
166
+ # ^ -- position start of line here:
167
+ pos += 1
168
+ initial += 1
169
+ offset += 1
170
+ adjustTab = False
171
+ spaceAfterMarker = True
172
+ elif next_char == "\t":
173
+ spaceAfterMarker = True
174
+
175
+ if (state.bsCount[nextLine] + offset) % 4 == 3:
176
+ # ' >\t test '
177
+ # ^ -- position start of line here (tab has width==1)
178
+ pos += 1
179
+ initial += 1
180
+ offset += 1
181
+ adjustTab = False
182
+ else:
183
+ # ' >\t test '
184
+ # ^ -- position start of line here + shift bsCount slightly
185
+ # to make extra space appear
186
+ adjustTab = True
187
+
188
+ else:
189
+ spaceAfterMarker = False
190
+
191
+ oldBMarks.append(state.bMarks[nextLine])
192
+ state.bMarks[nextLine] = pos
193
+
194
+ while pos < max:
195
+ ch = state.src[pos]
196
+
197
+ if isStrSpace(ch):
198
+ if ch == "\t":
199
+ offset += (
200
+ 4
201
+ - (
202
+ offset
203
+ + state.bsCount[nextLine]
204
+ + (1 if adjustTab else 0)
205
+ )
206
+ % 4
207
+ )
208
+ else:
209
+ offset += 1
210
+ else:
211
+ break
212
+
213
+ pos += 1
214
+
215
+ lastLineEmpty = pos >= max
216
+
217
+ oldBSCount.append(state.bsCount[nextLine])
218
+ state.bsCount[nextLine] = (
219
+ state.sCount[nextLine] + 1 + (1 if spaceAfterMarker else 0)
220
+ )
221
+
222
+ oldSCount.append(state.sCount[nextLine])
223
+ state.sCount[nextLine] = offset - initial
224
+
225
+ oldTShift.append(state.tShift[nextLine])
226
+ state.tShift[nextLine] = pos - state.bMarks[nextLine]
227
+
228
+ nextLine += 1
229
+ continue
230
+
231
+ # Case 2: line is not inside the blockquote, and the last line was empty.
232
+ if lastLineEmpty:
233
+ break
234
+
235
+ # Case 3: another tag found.
236
+ terminate = False
237
+
238
+ for terminatorRule in terminatorRules:
239
+ if terminatorRule(state, nextLine, endLine, True):
240
+ terminate = True
241
+ break
242
+
243
+ if terminate:
244
+ # Quirk to enforce "hard termination mode" for paragraphs;
245
+ # normally if you call `tokenize(state, startLine, nextLine)`,
246
+ # paragraphs will look below nextLine for paragraph continuation,
247
+ # but if blockquote is terminated by another tag, they shouldn't
248
+ state.lineMax = nextLine
249
+
250
+ if state.blkIndent != 0:
251
+ # state.blkIndent was non-zero, we now set it to zero,
252
+ # so we need to re-calculate all offsets to appear as
253
+ # if indent wasn't changed
254
+ oldBMarks.append(state.bMarks[nextLine])
255
+ oldBSCount.append(state.bsCount[nextLine])
256
+ oldTShift.append(state.tShift[nextLine])
257
+ oldSCount.append(state.sCount[nextLine])
258
+ state.sCount[nextLine] -= state.blkIndent
259
+
260
+ break
261
+
262
+ oldBMarks.append(state.bMarks[nextLine])
263
+ oldBSCount.append(state.bsCount[nextLine])
264
+ oldTShift.append(state.tShift[nextLine])
265
+ oldSCount.append(state.sCount[nextLine])
266
+
267
+ # A negative indentation means that this is a paragraph continuation
268
+ #
269
+ state.sCount[nextLine] = -1
270
+
271
+ nextLine += 1
272
+
273
+ oldIndent = state.blkIndent
274
+ state.blkIndent = 0
275
+
276
+ # Detect GitHub-style alert marker on the first content line.
277
+ # Note: `startLine` here refers to the first content line of the
278
+ # blockquote, after the `>` prefix has already been stripped by the
279
+ # blockquote parser above (bMarks/tShift adjusted to skip `> `).
280
+ alert_kind = None
281
+ if state.md.options.get("alerts", False) and nextLine > startLine:
282
+ alert_kind = _detect_alert(state, startLine)
283
+
284
+ lines = [startLine, 0]
285
+
286
+ if alert_kind is not None:
287
+ # Emit alert tokens instead of blockquote tokens
288
+ alert_lower = alert_kind.lower()
289
+ token = state.push("alert_open", "div", 1)
290
+ token.markup = ">"
291
+ token.attrSet("class", f"markdown-alert markdown-alert-{alert_lower}")
292
+ token.map = lines
293
+ token.info = alert_kind
294
+ token.meta = {"kind": alert_kind}
295
+
296
+ # Emit a title paragraph: <p class="markdown-alert-title">Kind</p>
297
+ token = state.push("alert_title_open", "p", 1)
298
+ token.attrSet("class", "markdown-alert-title")
299
+ title_token = state.push("inline", "", 0)
300
+ title_token.content = alert_kind.capitalize()
301
+ title_token.children = []
302
+ token = state.push("alert_title_close", "p", -1)
303
+
304
+ # Skip the marker line (startLine) and tokenize from startLine + 1.
305
+ contentStart = startLine + 1
306
+ if contentStart < nextLine:
307
+ # tokenize() updates state.line to nextLine as part of its
308
+ # contract, consistent with the blockquote code path below.
309
+ state.md.block.tokenize(state, contentStart, nextLine)
310
+ else:
311
+ state.line = nextLine
312
+
313
+ token = state.push("alert_close", "div", -1)
314
+ token.markup = ">"
315
+ else:
316
+ token = state.push("blockquote_open", "blockquote", 1)
317
+ token.markup = ">"
318
+ token.map = lines
319
+
320
+ state.md.block.tokenize(state, startLine, nextLine)
321
+
322
+ token = state.push("blockquote_close", "blockquote", -1)
323
+ token.markup = ">"
324
+
325
+ state.lineMax = oldLineMax
326
+ state.parentType = oldParentType
327
+ # Update the opening token map for both alert and blockquote containers.
328
+ lines[1] = state.line
329
+
330
+ # Restore original tShift; this might not be necessary since the parser
331
+ # has already been here, but just to make sure we can do that.
332
+ for i, item in enumerate(oldTShift):
333
+ state.bMarks[i + startLine] = oldBMarks[i]
334
+ state.tShift[i + startLine] = item
335
+ state.sCount[i + startLine] = oldSCount[i]
336
+ state.bsCount[i + startLine] = oldBSCount[i]
337
+
338
+ state.blkIndent = oldIndent
339
+
340
+ return True
341
+
342
+
343
+ _ALERT_TYPES = {"NOTE", "TIP", "IMPORTANT", "WARNING", "CAUTION"}
344
+
345
+
346
+ def _detect_alert(state: StateBlock, startLine: int) -> str | None:
347
+ """Detect ``[!TYPE]`` on *startLine* (after ``>`` prefix has been stripped).
348
+
349
+ Returns the alert type string (e.g. ``"NOTE"``) or ``None``.
350
+ """
351
+ pos = state.bMarks[startLine] + state.tShift[startLine]
352
+ maximum = state.eMarks[startLine]
353
+ src = state.src
354
+
355
+ # Trim trailing whitespace
356
+ while maximum > pos and src[maximum - 1] in (" ", "\t"):
357
+ maximum -= 1
358
+
359
+ if maximum - pos < 4:
360
+ return None
361
+ if src[pos] != "[" or src[pos + 1] != "!":
362
+ return None
363
+ if src[maximum - 1] != "]":
364
+ return None
365
+ type_str = src[pos + 2 : maximum - 1].upper()
366
+ if type_str not in _ALERT_TYPES:
367
+ return None
368
+ return type_str
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/heading.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Atex heading (#, ##, ...)"""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+
7
+ from ..common.utils import isStrSpace
8
+ from .state_block import StateBlock
9
+
10
+ LOGGER = logging.getLogger(__name__)
11
+
12
+
13
+ def heading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
14
+ LOGGER.debug("entering heading: %s, %s, %s, %s", state, startLine, endLine, silent)
15
+
16
+ pos = state.bMarks[startLine] + state.tShift[startLine]
17
+ maximum = state.eMarks[startLine]
18
+
19
+ if state.is_code_block(startLine):
20
+ return False
21
+
22
+ ch: str | None = state.src[pos]
23
+
24
+ if ch != "#" or pos >= maximum:
25
+ return False
26
+
27
+ # count heading level
28
+ level = 1
29
+ pos += 1
30
+ try:
31
+ ch = state.src[pos]
32
+ except IndexError:
33
+ ch = None
34
+ while ch == "#" and pos < maximum and level <= 6:
35
+ level += 1
36
+ pos += 1
37
+ try:
38
+ ch = state.src[pos]
39
+ except IndexError:
40
+ ch = None
41
+
42
+ if level > 6 or (pos < maximum and not isStrSpace(ch)):
43
+ return False
44
+
45
+ if silent:
46
+ return True
47
+
48
+ # Let's cut tails like ' ### ' from the end of string
49
+
50
+ maximum = state.skipSpacesBack(maximum, pos)
51
+ tmp = state.skipCharsStrBack(maximum, "#", pos)
52
+ if tmp > pos and isStrSpace(state.src[tmp - 1]):
53
+ maximum = tmp
54
+
55
+ state.line = startLine + 1
56
+
57
+ token = state.push("heading_open", "h" + str(level), 1)
58
+ token.markup = "########"[:level]
59
+ token.map = [startLine, state.line]
60
+
61
+ token = state.push("inline", "", 0)
62
+ token.content = state.src[pos:maximum].strip()
63
+ token.map = [startLine, state.line]
64
+ token.children = []
65
+
66
+ token = state.push("heading_close", "h" + str(level), -1)
67
+ token.markup = "########"[:level]
68
+
69
+ return True
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/lheading.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # lheading (---, ==)
2
+ import logging
3
+
4
+ from .state_block import StateBlock
5
+
6
+ LOGGER = logging.getLogger(__name__)
7
+
8
+
9
+ def lheading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
10
+ LOGGER.debug("entering lheading: %s, %s, %s, %s", state, startLine, endLine, silent)
11
+
12
+ level = None
13
+ nextLine = startLine + 1
14
+ ruler = state.md.block.ruler
15
+ terminatorRules = ruler.getRules("paragraph")
16
+
17
+ if state.is_code_block(startLine):
18
+ return False
19
+
20
+ oldParentType = state.parentType
21
+ state.parentType = "paragraph" # use paragraph to match terminatorRules
22
+
23
+ # jump line-by-line until empty one or EOF
24
+ while nextLine < endLine and not state.isEmpty(nextLine):
25
+ # this would be a code block normally, but after paragraph
26
+ # it's considered a lazy continuation regardless of what's there
27
+ if state.sCount[nextLine] - state.blkIndent > 3:
28
+ nextLine += 1
29
+ continue
30
+
31
+ # Check for underline in setext header
32
+ if state.sCount[nextLine] >= state.blkIndent:
33
+ pos = state.bMarks[nextLine] + state.tShift[nextLine]
34
+ maximum = state.eMarks[nextLine]
35
+
36
+ if pos < maximum:
37
+ marker = state.src[pos]
38
+
39
+ if marker in ("-", "="):
40
+ pos = state.skipCharsStr(pos, marker)
41
+ pos = state.skipSpaces(pos)
42
+
43
+ # /* = */
44
+ if pos >= maximum:
45
+ level = 1 if marker == "=" else 2
46
+ break
47
+
48
+ # quirk for blockquotes, this line should already be checked by that rule
49
+ if state.sCount[nextLine] < 0:
50
+ nextLine += 1
51
+ continue
52
+
53
+ # Some tags can terminate paragraph without empty line.
54
+ terminate = False
55
+ for terminatorRule in terminatorRules:
56
+ if terminatorRule(state, nextLine, endLine, True):
57
+ terminate = True
58
+ break
59
+ if terminate:
60
+ break
61
+
62
+ nextLine += 1
63
+
64
+ if not level:
65
+ # Didn't find valid underline
66
+ return False
67
+
68
+ content = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
69
+
70
+ state.line = nextLine + 1
71
+
72
+ token = state.push("heading_open", "h" + str(level), 1)
73
+ token.markup = marker
74
+ token.map = [startLine, state.line]
75
+
76
+ token = state.push("inline", "", 0)
77
+ token.content = content
78
+ token.map = [startLine, state.line - 1]
79
+ token.children = []
80
+
81
+ token = state.push("heading_close", "h" + str(level), -1)
82
+ token.markup = marker
83
+
84
+ state.parentType = oldParentType
85
+
86
+ return True
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_block/reference.py ADDED
@@ -0,0 +1,235 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+
3
+ from ..common.utils import charCodeAt, isSpace, normalizeReference
4
+ from .state_block import StateBlock
5
+
6
+ LOGGER = logging.getLogger(__name__)
7
+
8
+
9
+ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) -> bool:
10
+ LOGGER.debug(
11
+ "entering reference: %s, %s, %s, %s", state, startLine, _endLine, silent
12
+ )
13
+
14
+ pos = state.bMarks[startLine] + state.tShift[startLine]
15
+ maximum = state.eMarks[startLine]
16
+ nextLine = startLine + 1
17
+
18
+ if state.is_code_block(startLine):
19
+ return False
20
+
21
+ if state.src[pos] != "[":
22
+ return False
23
+
24
+ string = state.src[pos : maximum + 1]
25
+
26
+ # string = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
27
+ maximum = len(string)
28
+
29
+ labelEnd = None
30
+ pos = 1
31
+ while pos < maximum:
32
+ ch = charCodeAt(string, pos)
33
+ if ch == 0x5B: # /* [ */
34
+ return False
35
+ elif ch == 0x5D: # /* ] */
36
+ labelEnd = pos
37
+ break
38
+ elif ch == 0x0A: # /* \n */
39
+ if (lineContent := getNextLine(state, nextLine)) is not None:
40
+ string += lineContent
41
+ maximum = len(string)
42
+ nextLine += 1
43
+ elif ch == 0x5C: # /* \ */
44
+ pos += 1
45
+ if (
46
+ pos < maximum
47
+ and charCodeAt(string, pos) == 0x0A
48
+ and (lineContent := getNextLine(state, nextLine)) is not None
49
+ ):
50
+ string += lineContent
51
+ maximum = len(string)
52
+ nextLine += 1
53
+ pos += 1
54
+
55
+ if (
56
+ labelEnd is None or labelEnd < 0 or charCodeAt(string, labelEnd + 1) != 0x3A
57
+ ): # /* : */
58
+ return False
59
+
60
+ # [label]: destination 'title'
61
+ # ^^^ skip optional whitespace here
62
+ pos = labelEnd + 2
63
+ while pos < maximum:
64
+ ch = charCodeAt(string, pos)
65
+ if ch == 0x0A:
66
+ if (lineContent := getNextLine(state, nextLine)) is not None:
67
+ string += lineContent
68
+ maximum = len(string)
69
+ nextLine += 1
70
+ elif isSpace(ch):
71
+ pass
72
+ else:
73
+ break
74
+ pos += 1
75
+
76
+ # [label]: destination 'title'
77
+ # ^^^^^^^^^^^ parse this
78
+ destRes = state.md.helpers.parseLinkDestination(string, pos, maximum)
79
+ if not destRes.ok:
80
+ return False
81
+
82
+ href = state.md.normalizeLink(destRes.str)
83
+ if not state.md.validateLink(href):
84
+ return False
85
+
86
+ pos = destRes.pos
87
+
88
+ # save cursor state, we could require to rollback later
89
+ destEndPos = pos
90
+ destEndLineNo = nextLine
91
+
92
+ # [label]: destination 'title'
93
+ # ^^^ skipping those spaces
94
+ start = pos
95
+ while pos < maximum:
96
+ ch = charCodeAt(string, pos)
97
+ if ch == 0x0A:
98
+ if (lineContent := getNextLine(state, nextLine)) is not None:
99
+ string += lineContent
100
+ maximum = len(string)
101
+ nextLine += 1
102
+ elif isSpace(ch):
103
+ pass
104
+ else:
105
+ break
106
+ pos += 1
107
+
108
+ # [label]: destination 'title'
109
+ # ^^^^^^^ parse this
110
+ titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, None)
111
+ while titleRes.can_continue:
112
+ if (lineContent := getNextLine(state, nextLine)) is None:
113
+ break
114
+ string += lineContent
115
+ pos = maximum
116
+ maximum = len(string)
117
+ nextLine += 1
118
+ titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, titleRes)
119
+
120
+ if pos < maximum and start != pos and titleRes.ok:
121
+ title = titleRes.str
122
+ pos = titleRes.pos
123
+ else:
124
+ title = ""
125
+ pos = destEndPos
126
+ nextLine = destEndLineNo
127
+
128
+ # skip trailing spaces until the rest of the line
129
+ while pos < maximum:
130
+ ch = charCodeAt(string, pos)
131
+ if not isSpace(ch):
132
+ break
133
+ pos += 1
134
+
135
+ if pos < maximum and charCodeAt(string, pos) != 0x0A and title:
136
+ # garbage at the end of the line after title,
137
+ # but it could still be a valid reference if we roll back
138
+ title = ""
139
+ pos = destEndPos
140
+ nextLine = destEndLineNo
141
+ while pos < maximum:
142
+ ch = charCodeAt(string, pos)
143
+ if not isSpace(ch):
144
+ break
145
+ pos += 1
146
+
147
+ if pos < maximum and charCodeAt(string, pos) != 0x0A:
148
+ # garbage at the end of the line
149
+ return False
150
+
151
+ label = normalizeReference(string[1:labelEnd])
152
+ if not label:
153
+ # CommonMark 0.20 disallows empty labels
154
+ return False
155
+
156
+ # Reference can not terminate anything. This check is for safety only.
157
+ if silent:
158
+ return True
159
+
160
+ if "references" not in state.env:
161
+ state.env["references"] = {}
162
+
163
+ state.line = nextLine
164
+
165
+ # note, this is not part of markdown-it JS, but is useful for renderers
166
+ if state.md.options.get("inline_definitions", False):
167
+ token = state.push("definition", "", 0)
168
+ token.meta = {
169
+ "id": label,
170
+ "title": title,
171
+ "url": href,
172
+ "label": string[1:labelEnd],
173
+ }
174
+ token.map = [startLine, state.line]
175
+
176
+ if label not in state.env["references"]:
177
+ state.env["references"][label] = {
178
+ "title": title,
179
+ "href": href,
180
+ "map": [startLine, state.line],
181
+ }
182
+ else:
183
+ state.env.setdefault("duplicate_refs", []).append(
184
+ {
185
+ "title": title,
186
+ "href": href,
187
+ "label": label,
188
+ "map": [startLine, state.line],
189
+ }
190
+ )
191
+
192
+ return True
193
+
194
+
195
+ def getNextLine(state: StateBlock, nextLine: int) -> None | str:
196
+ endLine = state.lineMax
197
+
198
+ if nextLine >= endLine or state.isEmpty(nextLine):
199
+ # empty line or end of input
200
+ return None
201
+
202
+ isContinuation = False
203
+
204
+ # this would be a code block normally, but after paragraph
205
+ # it's considered a lazy continuation regardless of what's there
206
+ if state.is_code_block(nextLine):
207
+ isContinuation = True
208
+
209
+ # quirk for blockquotes, this line should already be checked by that rule
210
+ if state.sCount[nextLine] < 0:
211
+ isContinuation = True
212
+
213
+ if not isContinuation:
214
+ terminatorRules = state.md.block.ruler.getRules("reference")
215
+ oldParentType = state.parentType
216
+ state.parentType = "reference"
217
+
218
+ # Some tags can terminate paragraph without empty line.
219
+ terminate = False
220
+ for terminatorRule in terminatorRules:
221
+ if terminatorRule(state, nextLine, endLine, True):
222
+ terminate = True
223
+ break
224
+
225
+ state.parentType = oldParentType
226
+
227
+ if terminate:
228
+ # terminated by another block
229
+ return None
230
+
231
+ pos = state.bMarks[nextLine] + state.tShift[nextLine]
232
+ maximum = state.eMarks[nextLine]
233
+
234
+ # max + 1 explicitly includes the newline
235
+ return state.src[pos : maximum + 1]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/__init__.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from typing import TYPE_CHECKING
15
+
16
+ from ...utils import _LazyModule
17
+ from ...utils.import_utils import define_import_structure
18
+
19
+
20
+ if TYPE_CHECKING:
21
+ from .configuration_altclip import *
22
+ from .modeling_altclip import *
23
+ from .processing_altclip import *
24
+ else:
25
+ import sys
26
+
27
+ _file = globals()["__file__"]
28
+ sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/configuration_altclip.py ADDED
@@ -0,0 +1,246 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
2
+ # This file was automatically generated from src/transformers/models/altclip/modular_altclip.py.
3
+ # Do NOT edit this file manually as any edits will be overwritten by the generation of
4
+ # the file from the modular. If any change should be done, please apply the change to the
5
+ # modular_altclip.py file directly. One of our CI enforces this.
6
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
7
+ # Copyright 2022 The BAAI Teams Authors and The HuggingFace Inc. team. All rights reserved.
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+ from huggingface_hub.dataclasses import strict
20
+
21
+ from ...configuration_utils import PreTrainedConfig
22
+ from ...utils import auto_docstring, logging
23
+
24
+
25
+ logger = logging.get_logger(__name__)
26
+
27
+
28
+ @auto_docstring(checkpoint="BAAI/AltCLIP")
29
+ @strict
30
+ class AltCLIPTextConfig(PreTrainedConfig):
31
+ r"""
32
+ project_dim (`int`, *optional*, defaults to 768):
33
+ The dimensions of the teacher model before the mapping layer.
34
+
35
+ Examples:
36
+
37
+ ```python
38
+ >>> from transformers import AltCLIPTextModel, AltCLIPTextConfig
39
+
40
+ >>> # Initializing a AltCLIPTextConfig with BAAI/AltCLIP style configuration
41
+ >>> configuration = AltCLIPTextConfig()
42
+
43
+ >>> # Initializing a AltCLIPTextModel (with random weights) from the BAAI/AltCLIP style configuration
44
+ >>> model = AltCLIPTextModel(configuration)
45
+
46
+ >>> # Accessing the model configuration
47
+ >>> configuration = model.config
48
+ ```"""
49
+
50
+ model_type = "altclip_text_model"
51
+ base_config_key = "text_config"
52
+
53
+ vocab_size: int = 250002
54
+ hidden_size: int = 1024
55
+ intermediate_size: int = 4096
56
+ num_hidden_layers: int = 24
57
+ num_attention_heads: int = 16
58
+ max_position_embeddings: int = 514
59
+ hidden_act: str = "gelu"
60
+ layer_norm_eps: float | None = 1e-5
61
+ initializer_range: float = 0.02
62
+ initializer_factor: float = 0.02
63
+ pad_token_id: int | None = 1
64
+ bos_token_id: int | None = 0
65
+ eos_token_id: int | None = 2
66
+ hidden_dropout_prob: int | float = 0.1
67
+ attention_probs_dropout_prob: int | float = 0
68
+ type_vocab_size: int = 1
69
+ project_dim: int = 768
70
+
71
+ def validate_architecture(self):
72
+ """Part of `@strict`-powered validation. Validates the architecture of the config."""
73
+ if self.hidden_size % self.num_attention_heads != 0:
74
+ raise ValueError(
75
+ f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention "
76
+ f"heads ({self.num_attention_heads})."
77
+ )
78
+
79
+
80
+ @auto_docstring(checkpoint="BAAI/AltCLIP")
81
+ @strict
82
+ class AltCLIPVisionConfig(PreTrainedConfig):
83
+ r"""
84
+ Example:
85
+
86
+ ```python
87
+ >>> from transformers import AltCLIPVisionConfig, AltCLIPVisionModel
88
+
89
+ >>> # Initializing a AltCLIPVisionConfig with BAAI/AltCLIP style configuration
90
+ >>> configuration = AltCLIPVisionConfig()
91
+
92
+ >>> # Initializing a AltCLIPVisionModel (with random weights) from the BAAI/AltCLIP style configuration
93
+ >>> model = AltCLIPVisionModel(configuration)
94
+
95
+ >>> # Accessing the model configuration
96
+ >>> configuration = model.config
97
+ ```"""
98
+
99
+ model_type = "altclip_vision_model"
100
+ base_config_key = "vision_config"
101
+
102
+ hidden_size: int = 768
103
+ intermediate_size: int = 3072
104
+ projection_dim: int = 512
105
+ num_hidden_layers: int = 12
106
+ num_attention_heads: int = 12
107
+ num_channels: int = 3
108
+ image_size: int | list[int] | tuple[int, int] | None = 224
109
+ patch_size: int | list[int] | tuple[int, int] | None = 32
110
+ hidden_act: str = "quick_gelu"
111
+ layer_norm_eps: float = 1e-5
112
+ attention_dropout: int | float | None = 0.0
113
+ initializer_range: float = 0.02
114
+ initializer_factor: float = 1.0
115
+
116
+ def validate_architecture(self):
117
+ """Part of `@strict`-powered validation. Validates the architecture of the config."""
118
+ if self.hidden_size % self.num_attention_heads != 0:
119
+ raise ValueError(
120
+ f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention "
121
+ f"heads ({self.num_attention_heads})."
122
+ )
123
+
124
+
125
+ @auto_docstring(checkpoint="BAAI/AltCLIP")
126
+ @strict
127
+ class AltCLIPConfig(PreTrainedConfig):
128
+ r"""
129
+ Example:
130
+
131
+ ```python
132
+ >>> from transformers import AltCLIPConfig, AltCLIPModel
133
+
134
+ >>> # Initializing a AltCLIPConfig with BAAI/AltCLIP style configuration
135
+ >>> configuration = AltCLIPConfig()
136
+
137
+ >>> # Initializing a AltCLIPModel (with random weights) from the BAAI/AltCLIP style configuration
138
+ >>> model = AltCLIPModel(configuration)
139
+
140
+ >>> # Accessing the model configuration
141
+ >>> configuration = model.config
142
+
143
+ >>> # We can also initialize a AltCLIPConfig from a AltCLIPTextConfig and a AltCLIPVisionConfig
144
+
145
+ >>> # Initializing a AltCLIPText and AltCLIPVision configuration
146
+ >>> config_text = AltCLIPTextConfig()
147
+ >>> config_vision = AltCLIPVisionConfig()
148
+
149
+ >>> config = AltCLIPConfig(text_config=config_text, vision_config=config_vision)
150
+ ```"""
151
+
152
+ model_type = "altclip"
153
+ sub_configs = {"text_config": AltCLIPTextConfig, "vision_config": AltCLIPVisionConfig}
154
+
155
+ text_config: dict | AltCLIPTextConfig | None = None
156
+ vision_config: dict | AltCLIPVisionConfig | None = None
157
+
158
+ projection_dim: int = 768
159
+ logit_scale_init_value: float | int | None = 2.6592
160
+ initializer_factor: float | None = 1.0
161
+
162
+ def __post_init__(self, **kwargs):
163
+ if self.text_config is None:
164
+ text_config = {}
165
+ logger.info("`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values.")
166
+ elif isinstance(self.text_config, AltCLIPTextConfig):
167
+ text_config = self.text_config.to_dict()
168
+ else:
169
+ text_config = self.text_config
170
+
171
+ if self.vision_config is None:
172
+ vision_config = {}
173
+ logger.info("`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values.")
174
+ elif isinstance(self.vision_config, AltCLIPVisionConfig):
175
+ vision_config = self.vision_config.to_dict()
176
+ else:
177
+ vision_config = self.vision_config
178
+
179
+ # For backward compatibility check keyword args
180
+ # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
181
+ # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
182
+ # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
183
+ text_config_dict = kwargs.pop("text_config_dict", None)
184
+ vision_config_dict = kwargs.pop("vision_config_dict", None)
185
+
186
+ if text_config_dict is not None:
187
+ # This is the complete result when using `text_config_dict`.
188
+ _text_config_dict = AltCLIPTextConfig(**text_config_dict).to_dict()
189
+
190
+ # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
191
+ for key, value in _text_config_dict.items():
192
+ if key in text_config and value != text_config[key] and key != "transformers_version":
193
+ # If specified in `text_config_dict`
194
+ if key in text_config_dict:
195
+ message = (
196
+ f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
197
+ f'The value `text_config_dict["{key}"]` will be used instead.'
198
+ )
199
+ # If inferred from default argument values (just to be super careful)
200
+ else:
201
+ message = (
202
+ f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
203
+ f'value `text_config["{key}"]` will be overridden.'
204
+ )
205
+ logger.info(message)
206
+
207
+ # Update all values in `text_config` with the ones in `_text_config_dict`.
208
+ text_config.update(_text_config_dict)
209
+
210
+ if vision_config_dict is not None:
211
+ # This is the complete result when using `vision_config_dict`.
212
+ _vision_config_dict = AltCLIPVisionConfig(**vision_config_dict).to_dict()
213
+ # convert keys to string instead of integer
214
+ if "id2label" in _vision_config_dict:
215
+ _vision_config_dict["id2label"] = {
216
+ str(key): value for key, value in _vision_config_dict["id2label"].items()
217
+ }
218
+
219
+ # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
220
+ for key, value in _vision_config_dict.items():
221
+ if key in vision_config and value != vision_config[key] and key != "transformers_version":
222
+ # If specified in `vision_config_dict`
223
+ if key in vision_config_dict:
224
+ message = (
225
+ f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
226
+ f'values. The value `vision_config_dict["{key}"]` will be used instead.'
227
+ )
228
+ # If inferred from default argument values (just to be super careful)
229
+ else:
230
+ message = (
231
+ f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
232
+ f'The value `vision_config["{key}"]` will be overridden.'
233
+ )
234
+ logger.info(message)
235
+
236
+ # Update all values in `vision_config` with the ones in `_vision_config_dict`.
237
+ vision_config.update(_vision_config_dict)
238
+
239
+ # Finally we can convert back our unified text/vision configs to `PretrainedConfig`
240
+ self.text_config = AltCLIPTextConfig(**text_config)
241
+ self.vision_config = AltCLIPVisionConfig(**vision_config)
242
+
243
+ super().__post_init__(**kwargs)
244
+
245
+
246
+ __all__ = ["AltCLIPTextConfig", "AltCLIPVisionConfig", "AltCLIPConfig"]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/altclip/processing_altclip.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022 WenXiang ZhongzhiCheng LedellWu LiuGuang BoWenZhang The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """
15
+ Image/Text processor class for AltCLIP
16
+ """
17
+
18
+ from ...processing_utils import ProcessorMixin
19
+ from ...utils import auto_docstring
20
+
21
+
22
+ @auto_docstring
23
+ class AltCLIPProcessor(ProcessorMixin):
24
+ def __init__(self, image_processor=None, tokenizer=None):
25
+ super().__init__(image_processor, tokenizer)
26
+
27
+
28
+ __all__ = ["AltCLIPProcessor"]
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_003000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa6d3de70bd52b21c01abd539fe39af93889121f3ac7bffb10d4cafa55882001
3
+ size 927700322
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_179000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbc0a9693f5ee8aec606d5b7c2dfa61a4723f697272c48d06e3393ea93687f4d
3
+ size 927700322
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_185000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:60f25cc52272898fec4310310b99b1c39b8a05c025b64c1cc37107ca1d354afb
3
+ size 927700322
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_192000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d90c3432ab622266946dd0cd272e89740e6f41214d5aab98a170b9172d88f8e2
3
+ size 927700322
LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr4e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260610_020108/step_279000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:64654fc51957676402ac4ab478eec65183ad55382beac9e45ff6df293bab67be
3
+ size 927700322