Upload pasita.py with huggingface_hub
Browse files
pasita.py
CHANGED
|
@@ -312,6 +312,38 @@ def _space_heading_markers(text: str) -> str:
|
|
| 312 |
return re.sub(r"(?m)^(\s*#{1,6})(?=[^\s#])", r"\1 ", text)
|
| 313 |
|
| 314 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
|
| 316 |
paras = re.split(r"\n\s*\n", text.strip())
|
| 317 |
if len(paras) < 2:
|
|
@@ -501,11 +533,8 @@ def _repair_headings(text: str, input_text: str, max_lines: int = 80) -> str:
|
|
| 501 |
if checked >= max_lines:
|
| 502 |
break
|
| 503 |
checked += 1
|
| 504 |
-
first = body[0].lower()
|
| 505 |
best_ratio, best_src = 0.0, None
|
| 506 |
for src in source_lines:
|
| 507 |
-
if not src or src[0].lower() != first:
|
| 508 |
-
continue
|
| 509 |
if abs(len(src) - len(body)) > max(12, len(body) // 2):
|
| 510 |
continue
|
| 511 |
ratio = difflib.SequenceMatcher(None, body.lower(), src.lower()).ratio()
|
|
@@ -547,6 +576,8 @@ def postprocess(raw: str, input_text: str | None = None) -> str:
|
|
| 547 |
text = _repair_headings(text, input_text or "")
|
| 548 |
text = _fix_double_heading(text)
|
| 549 |
text = _space_heading_markers(text)
|
|
|
|
|
|
|
| 550 |
text = _cut_after_completion(text, input_text or "")
|
| 551 |
text = _cut_repeated_tail_paragraphs(text, input_text or "")
|
| 552 |
text = _cut_repeated_line_blocks(text)
|
|
@@ -665,6 +696,9 @@ def gfm_stats(text: str) -> dict[str, Any]:
|
|
| 665 |
malformed_links = max(0, link_opens - valid_links)
|
| 666 |
if text.count("](") > valid_links and re.search(r"\]\([^)]*$", text, re.M):
|
| 667 |
malformed_links = max(malformed_links, 1)
|
|
|
|
|
|
|
|
|
|
| 668 |
fences = FENCE_RE.findall(text)
|
| 669 |
fences_even = len(fences) % 2 == 0
|
| 670 |
clean = 1.0
|
|
@@ -674,12 +708,14 @@ def gfm_stats(text: str) -> dict[str, Any]:
|
|
| 674 |
clean -= 0.5
|
| 675 |
if malformed_links:
|
| 676 |
clean -= 0.25 * min(4, malformed_links)
|
|
|
|
|
|
|
| 677 |
if not fences_even:
|
| 678 |
clean -= 0.25
|
| 679 |
return {
|
| 680 |
"html_tokens": html_tokens,
|
| 681 |
"malformed_tables": malformed,
|
| 682 |
-
"malformed_links": malformed_links,
|
| 683 |
"fences_even": fences_even,
|
| 684 |
"gfm_clean": max(0.0, clean),
|
| 685 |
}
|
|
@@ -992,10 +1028,11 @@ def _is_separator_row(cells: list[str]) -> bool:
|
|
| 992 |
return bool(cells) and all(_SEPARATOR_CELL_RE.match(c) for c in cells)
|
| 993 |
|
| 994 |
|
| 995 |
-
def
|
| 996 |
lines = [l for l in text.splitlines() if l.strip()]
|
| 997 |
if len(lines) < 3:
|
| 998 |
-
return
|
|
|
|
| 999 |
for delim in (",", ";", "\t", "|", " "):
|
| 1000 |
for start in range(0, min(2, len(lines))):
|
| 1001 |
seg = lines[start:]
|
|
@@ -1003,32 +1040,45 @@ def _find_tabular_block(text: str):
|
|
| 1003 |
continue
|
| 1004 |
counts = [len(_split_cells(l, delim)) for l in seg]
|
| 1005 |
if min(counts) >= 2 and max(counts) == min(counts):
|
| 1006 |
-
|
| 1007 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1008 |
|
| 1009 |
|
| 1010 |
def delimited_to_markdown_table(text: str) -> str | None:
|
| 1011 |
-
found =
|
| 1012 |
if not found:
|
| 1013 |
return None
|
| 1014 |
-
|
| 1015 |
-
|
| 1016 |
-
|
| 1017 |
-
|
| 1018 |
-
|
| 1019 |
-
|
| 1020 |
-
|
| 1021 |
-
|
| 1022 |
-
|
| 1023 |
-
|
| 1024 |
-
|
| 1025 |
-
|
| 1026 |
-
out
|
| 1027 |
-
|
| 1028 |
-
|
| 1029 |
-
|
| 1030 |
-
out.append("| " + " | ".join(
|
| 1031 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1032 |
|
| 1033 |
|
| 1034 |
def infer_kind(text: str) -> str:
|
|
@@ -1052,6 +1102,9 @@ def structured_candidates(text: str) -> list[str]:
|
|
| 1052 |
return cands
|
| 1053 |
|
| 1054 |
|
|
|
|
|
|
|
|
|
|
| 1055 |
def convert_to_markdown(
|
| 1056 |
model,
|
| 1057 |
tokenizer,
|
|
@@ -1065,6 +1118,25 @@ def convert_to_markdown(
|
|
| 1065 |
) -> dict[str, Any]:
|
| 1066 |
if kind == "generic":
|
| 1067 |
kind = infer_kind(text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1068 |
if skip_model_for_structured and structured_fallback and kind in ("html", "data"):
|
| 1069 |
candidates = [postprocess(c, text) for c in structured_candidates(text)]
|
| 1070 |
if not candidates:
|
|
|
|
| 312 |
return re.sub(r"(?m)^(\s*#{1,6})(?=[^\s#])", r"\1 ", text)
|
| 313 |
|
| 314 |
|
| 315 |
+
def _promote_plain_headings(text: str, input_text: str) -> str:
|
| 316 |
+
if not input_text or "##" not in text and "###" not in text:
|
| 317 |
+
return text
|
| 318 |
+
in_lines = {_norm_cmp(l) for l in input_text.splitlines() if l.strip()}
|
| 319 |
+
lines = text.splitlines()
|
| 320 |
+
out = list(lines)
|
| 321 |
+
for i, line in enumerate(lines):
|
| 322 |
+
stripped = line.strip()
|
| 323 |
+
if not stripped or len(stripped) >= 60 or len(stripped) < 3:
|
| 324 |
+
continue
|
| 325 |
+
if re.match(r"^\s*(#{1,6}\s+|[-*•]\s+|\d+[.)]\s+|\|)", line):
|
| 326 |
+
continue
|
| 327 |
+
if re.search(r"[.?!:;]$", stripped):
|
| 328 |
+
continue
|
| 329 |
+
if _norm_cmp(stripped) not in in_lines:
|
| 330 |
+
continue
|
| 331 |
+
prev_blank = i == 0 or not lines[i - 1].strip()
|
| 332 |
+
next_line = lines[i + 1] if i + 1 < len(lines) else ""
|
| 333 |
+
next_blank = not next_line.strip()
|
| 334 |
+
next_is_list = bool(re.match(r"^\s*(?:[-*•]|\d+[.)])\s+\S", next_line))
|
| 335 |
+
has_content_after = any(l.strip() for l in lines[i + 1 :])
|
| 336 |
+
if prev_blank and (next_blank or next_is_list) and has_content_after:
|
| 337 |
+
out[i] = "## " + stripped
|
| 338 |
+
return "\n".join(out)
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def _balance_fences(text: str) -> str:
|
| 342 |
+
if text.count("```") % 2 == 1:
|
| 343 |
+
return text.rstrip() + "\n```"
|
| 344 |
+
return text
|
| 345 |
+
|
| 346 |
+
|
| 347 |
def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
|
| 348 |
paras = re.split(r"\n\s*\n", text.strip())
|
| 349 |
if len(paras) < 2:
|
|
|
|
| 533 |
if checked >= max_lines:
|
| 534 |
break
|
| 535 |
checked += 1
|
|
|
|
| 536 |
best_ratio, best_src = 0.0, None
|
| 537 |
for src in source_lines:
|
|
|
|
|
|
|
| 538 |
if abs(len(src) - len(body)) > max(12, len(body) // 2):
|
| 539 |
continue
|
| 540 |
ratio = difflib.SequenceMatcher(None, body.lower(), src.lower()).ratio()
|
|
|
|
| 576 |
text = _repair_headings(text, input_text or "")
|
| 577 |
text = _fix_double_heading(text)
|
| 578 |
text = _space_heading_markers(text)
|
| 579 |
+
text = _promote_plain_headings(text, input_text or "")
|
| 580 |
+
text = _balance_fences(text)
|
| 581 |
text = _cut_after_completion(text, input_text or "")
|
| 582 |
text = _cut_repeated_tail_paragraphs(text, input_text or "")
|
| 583 |
text = _cut_repeated_line_blocks(text)
|
|
|
|
| 696 |
malformed_links = max(0, link_opens - valid_links)
|
| 697 |
if text.count("](") > valid_links and re.search(r"\]\([^)]*$", text, re.M):
|
| 698 |
malformed_links = max(malformed_links, 1)
|
| 699 |
+
open_bracket_links = len(re.findall(r"\[[^\]\n]*https?://[^\]\n]*$", text, re.M))
|
| 700 |
+
open_bracket_links += len(re.findall(r"\[[^\]\n]*https?://[^\]\)\n]*[ \n]", text))
|
| 701 |
+
open_bracket_links = min(open_bracket_links, 4)
|
| 702 |
fences = FENCE_RE.findall(text)
|
| 703 |
fences_even = len(fences) % 2 == 0
|
| 704 |
clean = 1.0
|
|
|
|
| 708 |
clean -= 0.5
|
| 709 |
if malformed_links:
|
| 710 |
clean -= 0.25 * min(4, malformed_links)
|
| 711 |
+
if open_bracket_links:
|
| 712 |
+
clean -= 0.15 * open_bracket_links
|
| 713 |
if not fences_even:
|
| 714 |
clean -= 0.25
|
| 715 |
return {
|
| 716 |
"html_tokens": html_tokens,
|
| 717 |
"malformed_tables": malformed,
|
| 718 |
+
"malformed_links": malformed_links + open_bracket_links,
|
| 719 |
"fences_even": fences_even,
|
| 720 |
"gfm_clean": max(0.0, clean),
|
| 721 |
}
|
|
|
|
| 1028 |
return bool(cells) and all(_SEPARATOR_CELL_RE.match(c) for c in cells)
|
| 1029 |
|
| 1030 |
|
| 1031 |
+
def _find_tabular_blocks(text: str):
|
| 1032 |
lines = [l for l in text.splitlines() if l.strip()]
|
| 1033 |
if len(lines) < 3:
|
| 1034 |
+
return []
|
| 1035 |
+
found = []
|
| 1036 |
for delim in (",", ";", "\t", "|", " "):
|
| 1037 |
for start in range(0, min(2, len(lines))):
|
| 1038 |
seg = lines[start:]
|
|
|
|
| 1040 |
continue
|
| 1041 |
counts = [len(_split_cells(l, delim)) for l in seg]
|
| 1042 |
if min(counts) >= 2 and max(counts) == min(counts):
|
| 1043 |
+
found.append((delim, lines[:start], seg))
|
| 1044 |
+
break
|
| 1045 |
+
return found
|
| 1046 |
+
|
| 1047 |
+
|
| 1048 |
+
def _find_tabular_block(text: str):
|
| 1049 |
+
found = _find_tabular_blocks(text)
|
| 1050 |
+
return found[0] if found else None
|
| 1051 |
|
| 1052 |
|
| 1053 |
def delimited_to_markdown_table(text: str) -> str | None:
|
| 1054 |
+
found = _find_tabular_blocks(text)
|
| 1055 |
if not found:
|
| 1056 |
return None
|
| 1057 |
+
in_numbers = extract_number_keys(text)
|
| 1058 |
+
best, best_score = None, None
|
| 1059 |
+
for delim, pre, seg in found:
|
| 1060 |
+
rows = [_split_cells(l, delim) for l in seg]
|
| 1061 |
+
rows = [r for r in rows if not _is_separator_row(r)]
|
| 1062 |
+
if not rows:
|
| 1063 |
+
continue
|
| 1064 |
+
width = len(rows[0])
|
| 1065 |
+
if width < 2 or any(len(r) != width for r in rows):
|
| 1066 |
+
continue
|
| 1067 |
+
def esc(cells: list[str]) -> list[str]:
|
| 1068 |
+
return [c.replace("|", "\\|") for c in cells]
|
| 1069 |
+
out: list[str] = []
|
| 1070 |
+
if pre:
|
| 1071 |
+
out.append("\n".join(pre).strip())
|
| 1072 |
+
out.append("| " + " | ".join(esc(rows[0])) + " |")
|
| 1073 |
+
out.append("| " + " | ".join(["---"] * width) + " |")
|
| 1074 |
+
for r in rows[1:]:
|
| 1075 |
+
out.append("| " + " | ".join(esc(r)) + " |")
|
| 1076 |
+
table = "\n".join(out)
|
| 1077 |
+
out_numbers = extract_number_keys(table)
|
| 1078 |
+
score = (len(in_numbers & out_numbers), -len(out_numbers - in_numbers))
|
| 1079 |
+
if best_score is None or score > best_score:
|
| 1080 |
+
best, best_score = table, score
|
| 1081 |
+
return best
|
| 1082 |
|
| 1083 |
|
| 1084 |
def infer_kind(text: str) -> str:
|
|
|
|
| 1102 |
return cands
|
| 1103 |
|
| 1104 |
|
| 1105 |
+
LONG_PROMPT_TOKENS = 512
|
| 1106 |
+
|
| 1107 |
+
|
| 1108 |
def convert_to_markdown(
|
| 1109 |
model,
|
| 1110 |
tokenizer,
|
|
|
|
| 1118 |
) -> dict[str, Any]:
|
| 1119 |
if kind == "generic":
|
| 1120 |
kind = infer_kind(text)
|
| 1121 |
+
if kind == "generic" and skip_model_for_structured:
|
| 1122 |
+
prompt_len = len(tokenizer(build_prompt(text), add_special_tokens=True)["input_ids"])
|
| 1123 |
+
if prompt_len > LONG_PROMPT_TOKENS:
|
| 1124 |
+
chosen = _passthrough(text)
|
| 1125 |
+
metrics = score_text(text, chosen, must, kind)
|
| 1126 |
+
return {
|
| 1127 |
+
"markdown": chosen,
|
| 1128 |
+
"candidate_index": 0,
|
| 1129 |
+
"fallback_used": False,
|
| 1130 |
+
"route": "passthrough",
|
| 1131 |
+
"metrics": metrics,
|
| 1132 |
+
"candidate_metrics": [metrics],
|
| 1133 |
+
"candidates": [chosen],
|
| 1134 |
+
"raw_candidates": [],
|
| 1135 |
+
"prompt_tokens": prompt_len,
|
| 1136 |
+
"max_new_tokens": 0,
|
| 1137 |
+
"seconds": 0.0,
|
| 1138 |
+
"strategy": {"mode": "passthrough-long", "best_index": 0, "structured_fallback": structured_fallback, "route": "passthrough", "truncated_input": True},
|
| 1139 |
+
}
|
| 1140 |
if skip_model_for_structured and structured_fallback and kind in ("html", "data"):
|
| 1141 |
candidates = [postprocess(c, text) for c in structured_candidates(text)]
|
| 1142 |
if not candidates:
|