Update app.py

app.py

@@ -108,11 +108,11 @@ Primary goals for reconcile rate:
             earlier larger DA3 table — catches OCR-truncated description duplicates such as
             the BoA service-fee page where a VIDDYOZE -1.11 fragment table appears after the
             full service-fee table with a longer description string.
-- Fix 32: Headless amount column — PDFs often leave the rightmost money column without a header;
-            OCR emits an empty <th>. Fill with \"Amount\" when the column is mostly currency.
-            Second pass: collapse Date|Transaction Description|Additions|Subtractions + empty 4th
-            when OCR splits \"Preauth Debit\" / \"Pre-Auth Credit\" into its own column before
-            the merchant text (East West–style) — merge back to Date|Description|Amount.
+- Fix 32: Headless trailing <th> over currency cells — reuse the prior money header (Additions/
+            Subtractions) when present, else \"Amount\". Fix 32b aligns Date|Transaction Description|
+            Additions/Subtractions tables to the PDF: credits as Number|Date|Description|Additions
+            (blank Number when unused); debits as Date|Description|Subtractions (no Number); merges
+            OCR column splits. Adjacent duplicate HTML tables (same header + row multiset) are removed.
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -610,8 +610,25 @@ def _fill_headless_trailing_amount_headers(grid):
         if (amt_hits / total) < 0.45:
             continue
 
-        # Right-align column is the numeric amount; PDF often omits a label for it.
-        new_header[j] = "Amount"
+        left = j - 1
+        while left >= 0 and not new_header[left]:
+            left -= 1
+        prev = (new_header[left] if left >= 0 else "").strip()
+        pl = prev.lower()
+        if any(
+            k in pl
+            for k in (
+                "addition",
+                "subtraction",
+                "credit",
+                "debit",
+                "deposit",
+                "withdrawal",
+            )
+        ):
+            new_header[j] = prev
+        else:
+            new_header[j] = "Amount"
 
     grid[0] = new_header
     return grid
@@ -622,38 +639,84 @@ _MMDD_DATE_RE = re.compile(r"^\d{1,2}[-/]\d{2}(?:[-/]\d{2,4})?$")
 # Leading transaction-type cell when OCR splits category vs merchant (East West / similar).
 _STACKED_TXN_TYPE_RE = re.compile(
     r"^(?:Pre-Auth Credit|Pre-Auth Debit|Preauth Debit|"
-    r"Withdrawal Trns|Deposit(?:\s|$)|Outgoing Wire|Incoming Wire|"
+    r"Withdrawal Trns|Withdrawal(?:\s|$)|Deposit(?:\s|$)|Deposit Transfe|"
+    r"Outgoing Wire|Incoming Wire|"
     r"Service Charge|ACH Debit|ACH Credit|ACH Payment|ACH PMT|"
     r"Wire Transfer|POS Purchase|ATM Withdrawal|Transfer Trns)\b",
     re.IGNORECASE | re.DOTALL,
 )
 
 
-def _collapse_four_col_stacked_transaction_type_to_three(grid):
+def _normalize_date_desc_money_table_alignment(grid):
     """
-    Fix 32b: When the PDF is logically Date | one description | amount, OCR sometimes emits
-    four columns: Date | short type (e.g. Preauth Debit) | merchant tail | amount, with an
-    optional empty 4th header cell. Collapse to three columns so markdown matches the PDF row
-    semantics. Requires every non-empty data row to match the pattern (safe no-op otherwise).
+    Fix 32b: Align OCR tables with common bank PDF layouts.
+
+    * Credits (Additions): Number | Date | Transaction Description | Additions — Number is often
+      blank in the PDF but must exist as a column; OCR may omit it or split description across
+      columns and park amounts under a bogus \"Amount\" header.
+    * Debits (Subtractions): PDF is usually Date | Description | Subtractions (no Number) —
+      when OCR uses the same 4-column split error, merge back to three columns.
+
+    Handles (a) Date | short type | merchant tail | amount, (b) Date | full desc | empty | amount,
+    (c) continuation rows with a blank date. Per-row merge (no single global mode) so mixed OCR
+    errors (e.g. wrapped merchant lines) still normalize.
     """
     if not grid or len(grid) < 3:
         return grid
 
     header = [str(c or "").strip() for c in grid[0]]
-    if len(header) != 4:
-        return grid
+    n = len(header)
+
+    def _pad_row(cells, width):
+        cells = [str(c or "").strip() for c in cells]
+        while len(cells) < width:
+            cells.append("")
+        return cells[:width]
+
+    h0n = re.sub(r"\s+", " ", header[0]).strip().lower()
+    h1n = re.sub(r"\s+", " ", header[1]).strip().lower() if n > 1 else ""
+    h2n = re.sub(r"\s+", " ", header[2]).strip().lower() if n > 2 else ""
+    h3n = re.sub(r"\s+", " ", header[3]).strip().lower() if n > 3 else ""
+
+    money_label = header[2].strip() if n > 2 else ""
+    want_number_col = "addition" in h2n
+
+    # --- 3-column Date | Transaction Description | Additions/Subtractions: prepend Number for credits only ---
+    if n == 3:
+        if h0n != "date" or "transaction" not in h1n or "description" not in h1n:
+            return grid
+        if not any(
+            k in h2n
+            for k in (
+                "addition",
+                "subtraction",
+                "credit",
+                "debit",
+                "deposit",
+                "withdrawal",
+            )
+        ):
+            return grid
+        if not want_number_col:
+            return grid
+        out = [["Number", header[0], header[1], money_label or "Additions"]]
+        for row in grid[1:]:
+            c = _pad_row(row, 3)
+            if not any(c):
+                out.append(["", "", "", ""])
+                continue
+            out.append(["", c[0], c[1], c[2]])
+        return out
 
-    h0 = re.sub(r"\s+", " ", header[0]).strip().lower()
-    h1 = re.sub(r"\s+", " ", header[1]).strip().lower()
-    h2 = re.sub(r"\s+", " ", header[2]).strip().lower()
-    h3 = str(header[3] or "").strip().lower()
+    if n != 4:
+        return grid
 
-    if h0 != "date":
+    if h0n != "date":
         return grid
-    if "transaction" not in h1 or "description" not in h1:
+    if "transaction" not in h1n or "description" not in h1n:
         return grid
     if not any(
-        k in h2
+        k in h2n
         for k in (
             "addition",
             "subtraction",
@@ -664,58 +727,83 @@ def _collapse_four_col_stacked_transaction_type_to_three(grid):
         )
     ):
         return grid
-    if h3 not in ("", "amount") and not any(
-        k in h3 for k in ("addition", "subtraction", "credit", "debit")
+    if h3n not in ("", "amount") and not any(
+        k in h3n for k in ("addition", "subtraction", "credit", "debit")
     ):
         return grid
 
-    def _row_matches_split_pattern(cells):
-        while len(cells) < 4:
-            cells = cells + [""]
-        c0, c1, c2, c3 = (
-            str(cells[0] or "").strip(),
-            str(cells[1] or "").strip(),
-            str(cells[2] or "").strip(),
-            str(cells[3] or "").strip(),
-        )
-        if not c0 or not _MMDD_DATE_RE.match(c0):
-            return False
-        if not c1 or not _STACKED_TXN_TYPE_RE.match(c1):
-            return False
-        if not c2:
-            return False
-        if _is_amount_like(c2):
-            return False
-        if not _is_amount_like(c3):
-            return False
-        return True
-
     data_rows = [r for r in grid[1:] if any(str(c or "").strip() for c in r)]
-    if len(data_rows) < 2:
+    if not data_rows:
         return grid
 
+    has_date_led = False
     for row in data_rows:
-        cells = [str(c or "").strip() for c in row]
-        if not _row_matches_split_pattern(cells):
-            return grid
+        c = _pad_row(row, 4)
+        if c[0] and _MMDD_DATE_RE.match(c[0]):
+            has_date_led = True
+            break
+    if not has_date_led:
+        return grid
 
-    out = [
-        [
-            header[0],
-            header[1],
-            header[2],
-        ]
-    ]
+    def _merge_date_led_row(c0, c1, c2, c3):
+        split_ok = (
+            bool(c1)
+            and bool(_STACKED_TXN_TYPE_RE.match(c1))
+            and bool(c2)
+            and not _is_amount_like(c2)
+            and _is_amount_like(c3)
+        )
+        if split_ok:
+            return (c1 + " " + c2).strip(), c3
+        shift_ok = (
+            bool(c1)
+            and not _is_amount_like(c1)
+            and not (c2 or "").strip()
+            and _is_amount_like(c3)
+        )
+        if shift_ok:
+            return c1, c3
+        dep_ok = (
+            bool(c1)
+            and bool(_STACKED_TXN_TYPE_RE.match(c1))
+            and not (c2 or "").strip()
+            and _is_amount_like(c3)
+        )
+        if dep_ok:
+            return c1, c3
+        desc = ((c1 + " " + c2).strip() or c1).strip()
+        amt = c3 if _is_amount_like(c3) else ""
+        return desc, amt
+
+    ml = money_label or ("Additions" if want_number_col else "Subtractions")
+
+    if want_number_col:
+        out_h = ["Number", header[0], header[1], ml]
+    else:
+        out_h = [header[0], header[1], ml]
+
+    out = [out_h]
     for row in grid[1:]:
-        cells = [str(c or "").strip() for c in row]
-        while len(cells) < 4:
-            cells.append("")
-        if not any(cells):
-            out.append(["", "", ""])
+        c = _pad_row(row, 4)
+        c0, c1, c2, c3 = c[0], c[1], c[2], c[3]
+        if not any(c):
+            out.append([""] * len(out_h))
+            continue
+        if c0 and _MMDD_DATE_RE.match(c0):
+            desc, amt = _merge_date_led_row(c0, c1, c2, c3)
+            if want_number_col:
+                out.append(["", c0, desc, amt])
+            else:
+                out.append([c0, desc, amt])
             continue
-        d0, t1, tail, amt = cells[0], cells[1], cells[2], cells[3]
-        desc = (t1 + " " + tail).strip()
-        out.append([d0, desc, amt])
+        if want_number_col:
+            desc_t = (c1 + " " + c2).strip()
+            amt_t = c3 if _is_amount_like(c3) else ""
+            out.append(["", c0, desc_t, amt_t])
+        else:
+            desc_t = (c1 + " " + c2).strip()
+            amt_t = c3 if _is_amount_like(c3) else ""
+            out.append([c0, desc_t, amt_t])
 
     return out
 
@@ -2808,7 +2896,7 @@ def normalize_html_tables(text: str) -> str:
             grid = _merge_blank_header_text_columns(grid)
 
             grid = _fill_headless_trailing_amount_headers(grid)
-            grid = _collapse_four_col_stacked_transaction_type_to_three(grid)
+            grid = _normalize_date_desc_money_table_alignment(grid)
 
             grid = _reconstruct_separator_rows(grid)
 
@@ -4649,6 +4737,69 @@ def _strip_check_image_ocr_junk_html(text: str) -> str:
         return text
 
 
+def _dedupe_adjacent_duplicate_html_tables(text: str) -> str:
+    """
+    Drop consecutive <table> blocks that are identical after grid parse (same header + row multiset),
+    with only whitespace between them. Catches OCR echo where the same credits/debits grid is emitted twice.
+    """
+    if not text or "<table" not in text.lower():
+        return text
+    tbl_pat = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
+    matches = list(tbl_pat.finditer(text))
+    if len(matches) < 2:
+        return text
+
+    def _fingerprint(html: str):
+        try:
+            p = TableGridParser()
+            p.feed(html)
+            g = _build_grid(p.rows)
+            if not g or len(g) < 3:
+                return None
+            hdr = tuple(str(c or "").strip() for c in g[0])
+            body = []
+            for r in g[1:]:
+                if not any(str(c or "").strip() for c in r):
+                    continue
+                body.append(tuple(str(c or "").strip() for c in r))
+            if len(body) < 2:
+                return None
+            return (hdr, tuple(sorted(Counter(body).items())))
+        except Exception:
+            return None
+
+    def _gap_ok(ma, mb) -> bool:
+        gap = text[ma.end() : mb.start()]
+        return not re.search(r"<table", gap, re.IGNORECASE)
+
+    drop = set()
+    ref = 0
+    for j in range(1, len(matches)):
+        if j in drop:
+            continue
+        fp_r = _fingerprint(matches[ref].group(0))
+        fp_j = _fingerprint(matches[j].group(0))
+        if fp_r is not None and fp_j is not None and fp_r == fp_j and _gap_ok(
+            matches[ref], matches[j]
+        ):
+            drop.add(j)
+        else:
+            ref = j
+
+    if not drop:
+        return text
+
+    out: List[str] = []
+    pos = 0
+    for i, m in enumerate(matches):
+        out.append(text[pos : m.start()])
+        if i not in drop:
+            out.append(m.group(0))
+        pos = m.end()
+    out.append(text[pos:])
+    return "".join(out)
+
+
 def stabilize_tables_and_text(page_md: str) -> str:
     if not page_md:
         return page_md
@@ -4672,6 +4823,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
     stabilized = _strip_orphan_continued_da3_flatline(stabilized)
     stabilized = _strip_standalone_continued_banner_line(stabilized)
     stabilized = normalize_html_tables(stabilized)
+    stabilized = _dedupe_adjacent_duplicate_html_tables(stabilized)
     stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
@@ -4804,6 +4956,7 @@ def run_ocr(uploaded_file):
                     stabilized = _strip_orphan_continued_da3_flatline(stabilized)
                     stabilized = _strip_standalone_continued_banner_line(stabilized)
                     stabilized = normalize_html_tables(stabilized)
+                    stabilized = _dedupe_adjacent_duplicate_html_tables(stabilized)
                     stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
                     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
                     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)