OpceanAI commited on
Commit
f977aef
·
verified ·
1 Parent(s): 428ef1e

Upload pasita.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. pasita.py +99 -27
pasita.py CHANGED
@@ -312,6 +312,38 @@ def _space_heading_markers(text: str) -> str:
312
  return re.sub(r"(?m)^(\s*#{1,6})(?=[^\s#])", r"\1 ", text)
313
 
314
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
315
  def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
316
  paras = re.split(r"\n\s*\n", text.strip())
317
  if len(paras) < 2:
@@ -501,11 +533,8 @@ def _repair_headings(text: str, input_text: str, max_lines: int = 80) -> str:
501
  if checked >= max_lines:
502
  break
503
  checked += 1
504
- first = body[0].lower()
505
  best_ratio, best_src = 0.0, None
506
  for src in source_lines:
507
- if not src or src[0].lower() != first:
508
- continue
509
  if abs(len(src) - len(body)) > max(12, len(body) // 2):
510
  continue
511
  ratio = difflib.SequenceMatcher(None, body.lower(), src.lower()).ratio()
@@ -547,6 +576,8 @@ def postprocess(raw: str, input_text: str | None = None) -> str:
547
  text = _repair_headings(text, input_text or "")
548
  text = _fix_double_heading(text)
549
  text = _space_heading_markers(text)
 
 
550
  text = _cut_after_completion(text, input_text or "")
551
  text = _cut_repeated_tail_paragraphs(text, input_text or "")
552
  text = _cut_repeated_line_blocks(text)
@@ -665,6 +696,9 @@ def gfm_stats(text: str) -> dict[str, Any]:
665
  malformed_links = max(0, link_opens - valid_links)
666
  if text.count("](") > valid_links and re.search(r"\]\([^)]*$", text, re.M):
667
  malformed_links = max(malformed_links, 1)
 
 
 
668
  fences = FENCE_RE.findall(text)
669
  fences_even = len(fences) % 2 == 0
670
  clean = 1.0
@@ -674,12 +708,14 @@ def gfm_stats(text: str) -> dict[str, Any]:
674
  clean -= 0.5
675
  if malformed_links:
676
  clean -= 0.25 * min(4, malformed_links)
 
 
677
  if not fences_even:
678
  clean -= 0.25
679
  return {
680
  "html_tokens": html_tokens,
681
  "malformed_tables": malformed,
682
- "malformed_links": malformed_links,
683
  "fences_even": fences_even,
684
  "gfm_clean": max(0.0, clean),
685
  }
@@ -992,10 +1028,11 @@ def _is_separator_row(cells: list[str]) -> bool:
992
  return bool(cells) and all(_SEPARATOR_CELL_RE.match(c) for c in cells)
993
 
994
 
995
- def _find_tabular_block(text: str):
996
  lines = [l for l in text.splitlines() if l.strip()]
997
  if len(lines) < 3:
998
- return None
 
999
  for delim in (",", ";", "\t", "|", " "):
1000
  for start in range(0, min(2, len(lines))):
1001
  seg = lines[start:]
@@ -1003,32 +1040,45 @@ def _find_tabular_block(text: str):
1003
  continue
1004
  counts = [len(_split_cells(l, delim)) for l in seg]
1005
  if min(counts) >= 2 and max(counts) == min(counts):
1006
- return delim, lines[:start], seg
1007
- return None
 
 
 
 
 
 
1008
 
1009
 
1010
  def delimited_to_markdown_table(text: str) -> str | None:
1011
- found = _find_tabular_block(text)
1012
  if not found:
1013
  return None
1014
- delim, pre, seg = found
1015
- rows = [_split_cells(l, delim) for l in seg]
1016
- rows = [r for r in rows if not _is_separator_row(r)]
1017
- if not rows:
1018
- return None
1019
- width = len(rows[0])
1020
- if width < 2 or any(len(r) != width for r in rows):
1021
- return None
1022
- def esc(cells: list[str]) -> list[str]:
1023
- return [c.replace("|", "\\|") for c in cells]
1024
- out: list[str] = []
1025
- if pre:
1026
- out.append("\n".join(pre).strip())
1027
- out.append("| " + " | ".join(esc(rows[0])) + " |")
1028
- out.append("| " + " | ".join(["---"] * width) + " |")
1029
- for r in rows[1:]:
1030
- out.append("| " + " | ".join(esc(r)) + " |")
1031
- return "\n".join(out)
 
 
 
 
 
 
 
1032
 
1033
 
1034
  def infer_kind(text: str) -> str:
@@ -1052,6 +1102,9 @@ def structured_candidates(text: str) -> list[str]:
1052
  return cands
1053
 
1054
 
 
 
 
1055
  def convert_to_markdown(
1056
  model,
1057
  tokenizer,
@@ -1065,6 +1118,25 @@ def convert_to_markdown(
1065
  ) -> dict[str, Any]:
1066
  if kind == "generic":
1067
  kind = infer_kind(text)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1068
  if skip_model_for_structured and structured_fallback and kind in ("html", "data"):
1069
  candidates = [postprocess(c, text) for c in structured_candidates(text)]
1070
  if not candidates:
 
312
  return re.sub(r"(?m)^(\s*#{1,6})(?=[^\s#])", r"\1 ", text)
313
 
314
 
315
+ def _promote_plain_headings(text: str, input_text: str) -> str:
316
+ if not input_text or "##" not in text and "###" not in text:
317
+ return text
318
+ in_lines = {_norm_cmp(l) for l in input_text.splitlines() if l.strip()}
319
+ lines = text.splitlines()
320
+ out = list(lines)
321
+ for i, line in enumerate(lines):
322
+ stripped = line.strip()
323
+ if not stripped or len(stripped) >= 60 or len(stripped) < 3:
324
+ continue
325
+ if re.match(r"^\s*(#{1,6}\s+|[-*•]\s+|\d+[.)]\s+|\|)", line):
326
+ continue
327
+ if re.search(r"[.?!:;]$", stripped):
328
+ continue
329
+ if _norm_cmp(stripped) not in in_lines:
330
+ continue
331
+ prev_blank = i == 0 or not lines[i - 1].strip()
332
+ next_line = lines[i + 1] if i + 1 < len(lines) else ""
333
+ next_blank = not next_line.strip()
334
+ next_is_list = bool(re.match(r"^\s*(?:[-*•]|\d+[.)])\s+\S", next_line))
335
+ has_content_after = any(l.strip() for l in lines[i + 1 :])
336
+ if prev_blank and (next_blank or next_is_list) and has_content_after:
337
+ out[i] = "## " + stripped
338
+ return "\n".join(out)
339
+
340
+
341
+ def _balance_fences(text: str) -> str:
342
+ if text.count("```") % 2 == 1:
343
+ return text.rstrip() + "\n```"
344
+ return text
345
+
346
+
347
  def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
348
  paras = re.split(r"\n\s*\n", text.strip())
349
  if len(paras) < 2:
 
533
  if checked >= max_lines:
534
  break
535
  checked += 1
 
536
  best_ratio, best_src = 0.0, None
537
  for src in source_lines:
 
 
538
  if abs(len(src) - len(body)) > max(12, len(body) // 2):
539
  continue
540
  ratio = difflib.SequenceMatcher(None, body.lower(), src.lower()).ratio()
 
576
  text = _repair_headings(text, input_text or "")
577
  text = _fix_double_heading(text)
578
  text = _space_heading_markers(text)
579
+ text = _promote_plain_headings(text, input_text or "")
580
+ text = _balance_fences(text)
581
  text = _cut_after_completion(text, input_text or "")
582
  text = _cut_repeated_tail_paragraphs(text, input_text or "")
583
  text = _cut_repeated_line_blocks(text)
 
696
  malformed_links = max(0, link_opens - valid_links)
697
  if text.count("](") > valid_links and re.search(r"\]\([^)]*$", text, re.M):
698
  malformed_links = max(malformed_links, 1)
699
+ open_bracket_links = len(re.findall(r"\[[^\]\n]*https?://[^\]\n]*$", text, re.M))
700
+ open_bracket_links += len(re.findall(r"\[[^\]\n]*https?://[^\]\)\n]*[ \n]", text))
701
+ open_bracket_links = min(open_bracket_links, 4)
702
  fences = FENCE_RE.findall(text)
703
  fences_even = len(fences) % 2 == 0
704
  clean = 1.0
 
708
  clean -= 0.5
709
  if malformed_links:
710
  clean -= 0.25 * min(4, malformed_links)
711
+ if open_bracket_links:
712
+ clean -= 0.15 * open_bracket_links
713
  if not fences_even:
714
  clean -= 0.25
715
  return {
716
  "html_tokens": html_tokens,
717
  "malformed_tables": malformed,
718
+ "malformed_links": malformed_links + open_bracket_links,
719
  "fences_even": fences_even,
720
  "gfm_clean": max(0.0, clean),
721
  }
 
1028
  return bool(cells) and all(_SEPARATOR_CELL_RE.match(c) for c in cells)
1029
 
1030
 
1031
+ def _find_tabular_blocks(text: str):
1032
  lines = [l for l in text.splitlines() if l.strip()]
1033
  if len(lines) < 3:
1034
+ return []
1035
+ found = []
1036
  for delim in (",", ";", "\t", "|", " "):
1037
  for start in range(0, min(2, len(lines))):
1038
  seg = lines[start:]
 
1040
  continue
1041
  counts = [len(_split_cells(l, delim)) for l in seg]
1042
  if min(counts) >= 2 and max(counts) == min(counts):
1043
+ found.append((delim, lines[:start], seg))
1044
+ break
1045
+ return found
1046
+
1047
+
1048
+ def _find_tabular_block(text: str):
1049
+ found = _find_tabular_blocks(text)
1050
+ return found[0] if found else None
1051
 
1052
 
1053
  def delimited_to_markdown_table(text: str) -> str | None:
1054
+ found = _find_tabular_blocks(text)
1055
  if not found:
1056
  return None
1057
+ in_numbers = extract_number_keys(text)
1058
+ best, best_score = None, None
1059
+ for delim, pre, seg in found:
1060
+ rows = [_split_cells(l, delim) for l in seg]
1061
+ rows = [r for r in rows if not _is_separator_row(r)]
1062
+ if not rows:
1063
+ continue
1064
+ width = len(rows[0])
1065
+ if width < 2 or any(len(r) != width for r in rows):
1066
+ continue
1067
+ def esc(cells: list[str]) -> list[str]:
1068
+ return [c.replace("|", "\\|") for c in cells]
1069
+ out: list[str] = []
1070
+ if pre:
1071
+ out.append("\n".join(pre).strip())
1072
+ out.append("| " + " | ".join(esc(rows[0])) + " |")
1073
+ out.append("| " + " | ".join(["---"] * width) + " |")
1074
+ for r in rows[1:]:
1075
+ out.append("| " + " | ".join(esc(r)) + " |")
1076
+ table = "\n".join(out)
1077
+ out_numbers = extract_number_keys(table)
1078
+ score = (len(in_numbers & out_numbers), -len(out_numbers - in_numbers))
1079
+ if best_score is None or score > best_score:
1080
+ best, best_score = table, score
1081
+ return best
1082
 
1083
 
1084
  def infer_kind(text: str) -> str:
 
1102
  return cands
1103
 
1104
 
1105
+ LONG_PROMPT_TOKENS = 512
1106
+
1107
+
1108
  def convert_to_markdown(
1109
  model,
1110
  tokenizer,
 
1118
  ) -> dict[str, Any]:
1119
  if kind == "generic":
1120
  kind = infer_kind(text)
1121
+ if kind == "generic" and skip_model_for_structured:
1122
+ prompt_len = len(tokenizer(build_prompt(text), add_special_tokens=True)["input_ids"])
1123
+ if prompt_len > LONG_PROMPT_TOKENS:
1124
+ chosen = _passthrough(text)
1125
+ metrics = score_text(text, chosen, must, kind)
1126
+ return {
1127
+ "markdown": chosen,
1128
+ "candidate_index": 0,
1129
+ "fallback_used": False,
1130
+ "route": "passthrough",
1131
+ "metrics": metrics,
1132
+ "candidate_metrics": [metrics],
1133
+ "candidates": [chosen],
1134
+ "raw_candidates": [],
1135
+ "prompt_tokens": prompt_len,
1136
+ "max_new_tokens": 0,
1137
+ "seconds": 0.0,
1138
+ "strategy": {"mode": "passthrough-long", "best_index": 0, "structured_fallback": structured_fallback, "route": "passthrough", "truncated_input": True},
1139
+ }
1140
  if skip_model_for_structured and structured_fallback and kind in ("html", "data"):
1141
  candidates = [postprocess(c, text) for c in structured_candidates(text)]
1142
  if not candidates: