Update app.py

app.py

@@ -20,6 +20,9 @@ Primary goals for reconcile rate:
          (safe no-op for scanned PDFs and non-transaction pages)
 - Fix NF-1: extract Navy Federal "Summary of your deposit accounts" table from PDF text layer
             using spatial word positions (GLM-OCR fails on this wide-column layout)
+- Fix 13: collapse Date|Description|Amount|Description|Amount tables to 3 columns; fix fused
+            Beginning/Ending balance rows (UCB-style wide summary tables)
+- Post-pass: drop later duplicate subset Date|Description|Amount tables (repeated deposit blocks)
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -2695,6 +2698,96 @@ def _split_combined_summary_daily_balance_grid(grid):
     daily = [["Date", "Ledger Balance"]] + daily_rows
     return [summary, daily]
 
+def _normalize_double_desc_amount_header_table(grid):
+    """
+    Fix 13: Collapse OCR tables with a duplicated column group:
+      Date | Description | Amount | Description | Amount
+    Common on United Community Bank-style statements where a second
+    Description/Amount pair holds sub-totals (e.g. Average Ledger Balance).
+    Output: Date | Description | Amount
+
+    Strict guard: header row must match exactly those five labels (case/spacing
+    normalized). No bank name checks.
+    """
+    if not grid or len(grid) < 2:
+        return grid
+
+    raw_h = [str(c or "").strip() for c in grid[0]]
+    header_norm = [re.sub(r"\s+", " ", h).lower() for h in raw_h]
+    if len(header_norm) != 5:
+        return grid
+    if header_norm != ["date", "description", "amount", "description", "amount"]:
+        return grid
+
+    date_only_re = re.compile(r"^\d{1,2}/\d{1,2}/\d{4}$")
+    date_prefix_re = re.compile(r"^(\d{1,2}/\d{1,2}/\d{4})\s*(.*)$")
+    money_cell_re = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
+
+    def _is_money(s):
+        if not s or not str(s).strip():
+            return False
+        t = str(s).strip()
+        return bool(money_cell_re.match(t))
+
+    def _extract_money(s):
+        if not s:
+            return ""
+        m = re.search(r"(-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?)", str(s))
+        return m.group(1).strip() if m else ""
+
+    out = [["Date", "Description", "Amount"]]
+
+    for row in grid[1:]:
+        cells = [str(c or "").strip() for c in row]
+        while len(cells) < 5:
+            cells.append("")
+        c0, c1, c2, c3, c4 = cells[:5]
+
+        # Skip completely empty rows
+        if not any(cells[:5]):
+            continue
+
+        # A) Fused date + label in col0, amount in col1 (Beginning/Ending Balance, etc.)
+        mpre = date_prefix_re.match(c0)
+        if mpre and _is_money(c1):
+            date_s = mpre.group(1).strip()
+            tail = (mpre.group(2) or "").strip()
+            low = (c0 + " " + tail).lower()
+            if "beginning balance" in low:
+                desc = "Beginning Balance"
+            elif "ending balance" in low:
+                desc = "Ending Balance"
+            else:
+                desc = tail if tail else "Transaction"
+            out.append([date_s, desc, c1])
+            continue
+
+        # B) Date-only col0; description may contain Beginning Balance + amount; amount col wrong ($0)
+        if date_only_re.match(c0.strip()):
+            date_s = c0.strip()
+            desc = c1
+            amt = c2
+            if desc and "beginning balance" in desc.lower():
+                mx = _extract_money(desc)
+                if mx and (not amt or amt in ("$0.00", "0.00", "$0", ".00", "$.00")):
+                    amt = mx
+                    desc = re.sub(
+                        r"(?i)beginning\s+balance\s*\$?[\d,]+\.?\d*\s*",
+                        "Beginning Balance ",
+                        desc,
+                    )
+                    desc = re.sub(r"(?i)\s*average\s+ledger\s+balance\s*", " ", desc).strip()
+                    if not desc or desc == "Beginning Balance":
+                        desc = "Beginning Balance"
+            out.append([date_s, desc, amt])
+            continue
+
+        # C) Default: use first Date/Description/Amount triple; ignore empty second pair
+        out.append([c0, c1, c2])
+
+    return out if len(out) >= 2 else grid
+
+
 def _normalize_fused_date_posted_amount_table(grid):
     """
     Normalize 2-column OCR tables with fused header like:
@@ -2825,6 +2918,9 @@ def normalize_html_tables(text: str) -> str:
             grid = _drop_truly_empty_columns(grid)
             grid = _merge_blank_header_text_columns(grid)
 
+            # Fix 13: Date|Description|Amount|Description|Amount → 3 columns (UCB-style)
+            grid = _normalize_double_desc_amount_header_table(grid)
+
             # Fix 5: reconstruct mid-table separator rows split across columns by OCR
             grid = _reconstruct_separator_rows(grid)
 
@@ -2875,6 +2971,86 @@ def normalize_html_tables(text: str) -> str:
     out.append(text[last:])
     return "".join(out)
 
+def _dedupe_subset_transaction_tables_html(text: str) -> str:
+    """
+    Remove duplicate 3-column transaction tables where a later table's rows are a
+    strict subset of an earlier table's rows (same Date|Description|Amount header).
+    Used when OCR repeats the same deposit block under a section heading.
+    """
+    if not text or "<table" not in text.lower():
+        return text
+
+    pattern = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
+    matches = list(pattern.finditer(text))
+    if len(matches) < 2:
+        return text
+
+    def _parse_grid(html: str):
+        try:
+            p = TableGridParser()
+            p.feed(html)
+            return _build_grid(p.rows)
+        except Exception:
+            return None
+
+    grids = []
+    for m in matches:
+        grids.append((m, _parse_grid(m.group(0))))
+
+    drop_idx = set()
+    for i in range(1, len(grids)):
+        mi, gi = grids[i]
+        if not gi or len(gi) < 2:
+            continue
+        h = [re.sub(r"\s+", " ", str(c or "").strip().lower()) for c in gi[0]]
+        if len(h) != 3 or h != ["date", "description", "amount"]:
+            continue
+
+        for j in range(i):
+            if j in drop_idx:
+                continue
+            mj, gj = grids[j]
+            if not gj or len(gj) < 2:
+                continue
+            hj = [re.sub(r"\s+", " ", str(c or "").strip().lower()) for c in gj[0]]
+            if len(hj) != 3 or hj != ["date", "description", "amount"]:
+                continue
+
+            def _row_set(g):
+                s = set()
+                for r in g[1:]:
+                    if not any(str(c or "").strip() for c in r):
+                        continue
+                    t = tuple(str(c or "").strip() for c in r)
+                    if any(t):
+                        s.add(t)
+                return s
+
+            si, sj = _row_set(gi), _row_set(gj)
+            if not si or not sj:
+                continue
+            # Drop later table if it is a strict subset of an earlier one (repeated block).
+            if si <= sj and len(si) < len(sj):
+                drop_idx.add(i)
+                break
+            if si == sj:
+                drop_idx.add(i)
+                break
+
+    if not drop_idx:
+        return text
+
+    out = []
+    last = 0
+    for idx, m in enumerate(matches):
+        out.append(text[last : m.start()])
+        if idx not in drop_idx:
+            out.append(m.group(0))
+        last = m.end()
+    out.append(text[last:])
+    return "".join(out)
+
+
 def stabilize_tables_and_text(page_md: str) -> str:
     if not page_md:
         return page_md
@@ -2892,6 +3068,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
     stabilized = "\n\n".join(out_blocks)
     stabilized = convert_plaintext_bank_sections(stabilized)
     stabilized = normalize_html_tables(stabilized)
+    stabilized = _dedupe_subset_transaction_tables_html(stabilized)
     return close_unclosed_html(stabilized)
 
 # --------------------------