File size: 1,984 Bytes
52c13bd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | from .state_inline import StateInline
def fragments_join(state: StateInline) -> None:
"""
Clean up tokens after emphasis and strikethrough postprocessing:
merge adjacent text nodes into one and re-calculate all token levels
This is necessary because initially emphasis delimiter markers (``*, _, ~``)
are treated as their own separate text tokens. Then emphasis rule either
leaves them as text (needed to merge with adjacent text) or turns them
into opening/closing tags (which messes up levels inside).
"""
level = 0
maximum = len(state.tokens)
curr = last = 0
while curr < maximum:
# re-calculate levels after emphasis/strikethrough turns some text nodes
# into opening/closing tags
if state.tokens[curr].nesting < 0:
level -= 1 # closing tag
state.tokens[curr].level = level
if state.tokens[curr].nesting > 0:
level += 1 # opening tag
if (
state.tokens[curr].type == "text"
and curr + 1 < maximum
and state.tokens[curr + 1].type == "text"
):
# Collapse a run of adjacent text nodes in a single join, instead
# of pairwise `a + b` concatenation. The pairwise form is O(L*k)
# in the size of the run because each step rebuilds the growing
# prefix; "".join is O(L).
parts = [state.tokens[curr].content]
curr += 1
while curr < maximum and state.tokens[curr].type == "text":
parts.append(state.tokens[curr].content)
curr += 1
merged = state.tokens[curr - 1]
merged.content = "".join(parts)
merged.level = level
state.tokens[last] = merged
last += 1
continue
if curr != last:
state.tokens[last] = state.tokens[curr]
last += 1
curr += 1
if curr != last:
del state.tokens[last:]
|