Update app.py

app.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import re
 import subprocess
 import tempfile
 import time
@@ -150,15 +151,76 @@ def get_parser():
 def _extract_md(result: Any) -> str:
     if result is None:
         return ""
+
+    def _close_unbalanced_tables(chunk: str) -> str:
+        # Some OCR chunks end mid-table near page boundaries, which can cause
+        # subsequent content (including page separators) to render inside a cell.
+        opens = len(re.findall(r"<table\b", chunk, flags=re.IGNORECASE))
+        closes = len(re.findall(r"</table>", chunk, flags=re.IGNORECASE))
+        missing = max(0, opens - closes)
+        if missing:
+            chunk = chunk.rstrip() + ("\n</table>" * missing)
+        return chunk
+
     if isinstance(result, list):
         chunks = []
         for item in result:
             md = getattr(item, "markdown_result", "")
             if md:
-                chunks.append(str(md).strip())
+                chunks.append(_close_unbalanced_tables(str(md).strip()))
         return "\n\n---page-separator---\n\n".join([c for c in chunks if c]).strip()
     md = getattr(result, "markdown_result", "")
-    return str(md).strip() if md else ""
+    return _close_unbalanced_tables(str(md).strip()) if md else ""
+
+
+def _clean_markdown(md: str) -> str:
+    if not md:
+        return md
+
+    # Drop frequently repeated bank footer/header lines.
+    drop_line_patterns = [
+        r"Call 1-800-937-2000 for 24-hour Bank-by-Phone services.*",
+        r"Bank Deposits FDIC Insured \| TD Bank, N\.A\. \| Equal Housing Lender.*",
+        r"Go paperless\.",
+        r"Scan the QR code to opt in to paperless statements\.",
+    ]
+    drop_line_regexes = [re.compile(p, re.IGNORECASE) for p in drop_line_patterns]
+
+    cleaned_lines: List[str] = []
+    for line in md.splitlines():
+        s = line.strip()
+        if any(rx.fullmatch(s) for rx in drop_line_regexes):
+            continue
+        cleaned_lines.append(line)
+    md = "\n".join(cleaned_lines)
+
+    # Remove long legal notice section that repeats and is usually not needed
+    # for transaction extraction.
+    legal_start_markers = [
+        "## FOR CONSUMER ACCOUNTS ONLY",
+        "## FOR CONSUMER LOAN ACCOUNTS ONLY",
+        "## In case of Errors or Questions About Your Bill:",
+    ]
+    lines = md.splitlines()
+    kept: List[str] = []
+    skipping = False
+    for line in lines:
+        s = line.strip()
+        if any(s.startswith(m) for m in legal_start_markers):
+            skipping = True
+            continue
+        if skipping and s == "---page-separator---":
+            skipping = False
+            kept.append(line)
+            continue
+        if not skipping:
+            kept.append(line)
+    md = "\n".join(kept)
+
+    # Remove duplicate separators and excessive blank lines.
+    md = re.sub(r"(?:\n\s*){3,}", "\n\n", md)
+    md = re.sub(r"(?:\n\s*---page-separator---\s*\n){2,}", "\n\n---page-separator---\n\n", md)
+    return md.strip()
 
 
 def run_ocr(file_obj):
@@ -175,7 +237,7 @@ def run_ocr(file_obj):
             result = parser.parse(page_images)
         else:
             result = parser.parse(path)
-        md = _extract_md(result) or "(No content)"
+        md = _clean_markdown(_extract_md(result)) or "(No content)"
         return md
     except Exception as e:
         import traceback