Update app.py

app.py

@@ -119,6 +119,11 @@ Primary goals for reconcile rate:
             five columns using trailing amount in the description cell and TD-specific debit/credit
             routing. Fires on \"continued\" in-table text and/or TD body cues (TD ZELLE, LENDINGCLUB,
             etc.). Safe no-op elsewhere.
+- Fix MD-1: Markdown fidelity — (a) collapse adjacent HTML tables that are the same register
+            (identical multiset of date + debit + credit + balance per row, descriptions may differ);
+            keep the table with richer description text so output matches the PDF once. (b) Common OCR
+            typos (e.g. CINTHIAL→CINTHIA L) and double-escaped ampersands (&→&).
+            Gap between tables must not contain another <table> or page-separator.
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -139,7 +144,7 @@ import os
 import re
 import html
 import tempfile
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 from collections import Counter, defaultdict
 from html.parser import HTMLParser
 
@@ -4956,6 +4961,139 @@ def _strip_check_image_ocr_junk_html(text: str) -> str:
         return text
 
 
+def _fix_markdown_fidelity_typos_and_escapes(text: str) -> str:
+    """Fix MD-1: common OCR name glitches and double HTML-escaped ampersands in OCR markdown."""
+    if not text:
+        return text
+    text = re.sub(
+        r"\bCINTHIAL\s+BARTON\b",
+        "CINTHIA L BARTON",
+        text,
+        flags=re.IGNORECASE,
+    )
+    text = text.replace("&amp;amp;", "&amp;")
+    return text
+
+
+def _gap_ok_loose_register_pair(ma, mb, text: str) -> bool:
+    gap = text[ma.end() : mb.start()]
+    if re.search(r"<table", gap, re.IGNORECASE):
+        return False
+    if re.search(r"page-separator", gap, re.IGNORECASE):
+        return False
+    return True
+
+
+def _norm_register_amount_key(s: str) -> str:
+    t = str(s or "").strip().replace("$", "").replace(",", "").replace("−", "-")
+    if not t or t in ("---", "—"):
+        return ""
+    try:
+        return f"{float(t):.2f}"
+    except ValueError:
+        return t
+
+
+def _register_row_loose_key_cells(cells: List[str]) -> Optional[Tuple[str, str, str, str]]:
+    if len(cells) < 5:
+        return None
+    d = str(cells[0]).strip()
+    if not re.match(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$", d):
+        return None
+    deb = _norm_register_amount_key(cells[2])
+    cre = _norm_register_amount_key(cells[3])
+    bal = _norm_register_amount_key(cells[4])
+    return (d, deb, cre, bal)
+
+
+def _register_multiset_and_score(grid) -> Tuple[Optional[Counter], int]:
+    if not grid or len(grid) < 2:
+        return None, 0
+    ncols = max(len(r) for r in grid if isinstance(r, list))
+    if ncols < 5:
+        return None, 0
+    c = Counter()
+    desc_score = 0
+    for row in grid[1:]:
+        cells = [str(x or "").strip() for x in row]
+        while len(cells) < ncols:
+            cells.append("")
+        cells = cells[: max(5, ncols)]
+        if len(cells) < 5:
+            continue
+        k = _register_row_loose_key_cells(cells[:5])
+        if not k:
+            continue
+        c[k] += 1
+        desc_score += len(cells[1])
+    if sum(c.values()) < 3:
+        return None, 0
+    return c, desc_score
+
+
+def _dedupe_adjacent_loose_register_duplicate_tables(text: str) -> str:
+    """
+    Fix MD-1: Drop adjacent duplicate bank register <table> blocks when date+debit+credit+balance
+    multiset matches but description text differs (OCR double-emission). Keeps the richer table.
+    """
+    if not text or "<table" not in text.lower():
+        return text
+    try:
+        tbl_pat = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
+        matches = list(tbl_pat.finditer(text))
+        if len(matches) < 2:
+            return text
+
+        def _grid(m):
+            p = TableGridParser()
+            p.feed(m.group(0))
+            return _build_grid(p.rows)
+
+        keep_indices: List[int] = [0]
+        for j in range(1, len(matches)):
+            prev_i = keep_indices[-1]
+            prev_m, cur_m = matches[prev_i], matches[j]
+            if not _gap_ok_loose_register_pair(prev_m, cur_m, text):
+                keep_indices.append(j)
+                continue
+            ga, gb = _grid(prev_m), _grid(cur_m)
+            ma, sa = _register_multiset_and_score(ga)
+            mb, sb = _register_multiset_and_score(gb)
+            if ma is None or mb is None or ma != mb:
+                keep_indices.append(j)
+                continue
+            if sb >= sa:
+                keep_indices[-1] = j
+                log.info(
+                    "Fix MD-1: loose-dup register — kept table at offset %d (desc score %d vs %d)",
+                    cur_m.start(),
+                    sb,
+                    sa,
+                )
+            else:
+                log.info(
+                    "Fix MD-1: dropped loose-dup register table at offset %d",
+                    cur_m.start(),
+                )
+
+        drop = set(range(len(matches))) - set(keep_indices)
+        if not drop:
+            return text
+
+        out: List[str] = []
+        pos = 0
+        for i, m in enumerate(matches):
+            out.append(text[pos : m.start()])
+            if i not in drop:
+                out.append(m.group(0))
+            pos = m.end()
+        out.append(text[pos:])
+        return "".join(out)
+    except Exception as e:
+        log.warning("_dedupe_adjacent_loose_register_duplicate_tables failed: %s", e)
+        return text
+
+
 def _dedupe_adjacent_duplicate_html_tables(text: str) -> str:
     """
     Drop consecutive <table> blocks that are identical after grid parse (same header + row multiset),
@@ -5028,6 +5166,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
     _tl_freq_context.clear()
 
     page_md = normalize_money_glyphs(page_md)
+    page_md = _fix_markdown_fidelity_typos_and_escapes(page_md)
 
     blocks = re.split(r"\n\s*\n", page_md.strip())
     out_blocks = []
@@ -5043,6 +5182,8 @@ def stabilize_tables_and_text(page_md: str) -> str:
     stabilized = _strip_standalone_continued_banner_line(stabilized)
     stabilized = normalize_html_tables(stabilized)
     stabilized = _dedupe_adjacent_duplicate_html_tables(stabilized)
+    stabilized = _dedupe_adjacent_loose_register_duplicate_tables(stabilized)
+    stabilized = _fix_markdown_fidelity_typos_and_escapes(stabilized)
     stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
@@ -5054,6 +5195,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
     stabilized = _strip_orphan_continued_da3_flatline(stabilized)
     stabilized = _strip_standalone_continued_banner_line(stabilized)
     stabilized = _strip_check_image_ocr_junk_html(stabilized)  # Fix 31
+    stabilized = _fix_markdown_fidelity_typos_and_escapes(stabilized)
     return close_unclosed_html(stabilized)
 
 # --------------------------
@@ -5176,6 +5318,8 @@ def run_ocr(uploaded_file):
                     stabilized = _strip_standalone_continued_banner_line(stabilized)
                     stabilized = normalize_html_tables(stabilized)
                     stabilized = _dedupe_adjacent_duplicate_html_tables(stabilized)
+                    stabilized = _dedupe_adjacent_loose_register_duplicate_tables(stabilized)
+                    stabilized = _fix_markdown_fidelity_typos_and_escapes(stabilized)
                     stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
                     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
                     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
@@ -5254,6 +5398,9 @@ def run_ocr(uploaded_file):
         if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
             merged = _strip_orphan_continued_da3_flatline(merged)
             merged = _strip_standalone_continued_banner_line(merged)
+            merged = _dedupe_adjacent_duplicate_html_tables(merged)
+            merged = _dedupe_adjacent_loose_register_duplicate_tables(merged)
+            merged = _fix_markdown_fidelity_typos_and_escapes(merged)
             merged = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(merged)
             merged = _split_ucb_deposits_electronic_sections_html(merged)
             merged = _patch_da3_msft_g_ref_amounts_across_tables(merged)