Update app.py

app.py

@@ -43,6 +43,14 @@ Primary goals for reconcile rate:
             (then subset dedupe can drop the stray duplicate block, e.g. under Deposits).
 - Fix 22: Drop a single OCR “flat line” between pages: (continued) + DESCRIPTION + AMOUNT + many MM/DD
             tokens (noise before a proper <table>).
+- Fix 23: Doc-wide DA3 table dedupe using multiset of (date, amount) keys — catches duplicate sections when
+            OCR varies description text between copies (e.g. withdrawals repeated under Deposits).
+- Fix 24: Run orphan flatline strip on merged PDF output and after text-layer patch so page-boundary
+            noise is removed consistently.
+- Fix 25: After a centered “deposits + credits/interest” heading, if two DA3 tables appear back-to-back
+            and the first is mostly credit/deposit rows while the second is mostly debit/withdrawal-style
+            rows (generic keywords), drop the second — fixes OCR pasting the withdrawals block again.
+- Fix 23b: Loose multiset keys use normalized float amounts so comma/spacing variants still match.
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -64,7 +72,7 @@ import re
 import html
 import tempfile
 from typing import List, Tuple
-from collections import defaultdict
+from collections import Counter, defaultdict
 from html.parser import HTMLParser
 
 import yaml
@@ -3143,6 +3151,59 @@ def _transaction_row_dedupe_key(cells):
     return (date_s, desc_s, amt_s)
 
 
+def _da3_loose_key(cells):
+    """
+    (date, amount) only — for detecting duplicate DA3 tables when description drifts between OCR passes.
+    Uses strict currency + float amount so junk cells do not create keys; 1,234.56 vs 1234.56 match.
+    """
+    parts = [str(c or "").strip() for c in cells[:3]]
+    while len(parts) < 3:
+        parts.append("")
+    date_s, desc_s, amt_s = parts[0], parts[1], parts[2]
+    if not _DA3_LEADING_DATE.match(date_s):
+        return None
+    comb = f"{date_s} {desc_s}".lower()
+    if re.search(r"\btotal\b", comb) and len(desc_s) > 25:
+        return None
+    if not _DA3_STRICT_AMOUNT.match(amt_s.strip()):
+        return None
+    t_amt = re.sub(r"[\s$,]", "", amt_s).replace("−", "-").replace("$", "")
+    try:
+        af = round(float(t_amt), 2)
+    except ValueError:
+        return None
+    d_norm = re.sub(r"\s+", "", date_s)
+    return (d_norm, af)
+
+
+def _row_counter_loose_da3(g):
+    """Multiset of (date, amount) for DA3 data rows with strict amounts."""
+    c = Counter()
+    if not g or len(g) < 2:
+        return c
+    for r in g[1:]:
+        cells = [str(x or "").strip() for x in r]
+        if not any(cells):
+            continue
+        k = _da3_loose_key(cells)
+        if k:
+            c[k] += 1
+    return c
+
+
+def _overlap_ratio_counter(c_num, c_den):
+    """Fraction of c_num's multiset mass matched in c_den (min counts per key)."""
+    tot = sum(c_num.values())
+    if tot == 0:
+        return 0.0
+    hit = sum(min(c_num[k], c_den.get(k, 0)) for k in c_num)
+    return hit / tot
+
+
+def _multiset_leq(c_small, c_big):
+    return all(c_small[k] <= c_big.get(k, 0) for k in c_small)
+
+
 def _is_da3_header(grid):
     if not grid or len(grid[0]) < 3:
         return False
@@ -3400,12 +3461,15 @@ def _strip_orphan_continued_da3_flatline(text: str) -> str:
     out = []
     for line in lines:
         core = line.rstrip("\r\n")
+        date_hits = len(re.findall(r"\d{2}/\d{2}", core)) + len(
+            re.findall(r"\b\d{2}-\d{2}-\d{2}\b", core)
+        )
         if (
             len(core) > 80
             and "(continued)" in core.lower()
             and "description" in core.lower()
             and "amount" in core.lower()
-            and len(re.findall(r"\d{2}/\d{2}", core)) >= 3
+            and date_hits >= 3
         ):
             continue
         out.append(line)
@@ -3570,6 +3634,8 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
         "Deposits (continued)" preview before the same rows appear in the full list).
       - Identical row sets → drop the later table.
       - Fuzzy overlap (≥92% of later rows match earlier) → drop later (Electronic Debits vs main).
+      - Fix 23: multiset overlap on (date, amount) only — drop when OCR varies description
+        between two copies of the same section (e.g. withdrawals pasted twice).
       - Runs multiple passes until stable.
     Row matching uses normalized keys so OCR variants (e.g. "0 27" vs "027", backticks)
     do not prevent duplicate tables from being detected.
@@ -3653,6 +3719,23 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
                     drop_idx.add(j)
                     continue
 
+                # Fix 23: multiset (date, amount) overlap — duplicate tables when description OCR differs.
+                ci = _row_counter_loose_da3(gi)
+                cj = _row_counter_loose_da3(gj)
+                ni, nj = sum(ci.values()), sum(cj.values())
+                if ni >= 6 and nj >= 6:
+                    oi_j = _overlap_ratio_counter(ci, cj)
+                    oj_i = _overlap_ratio_counter(cj, ci)
+                    if oi_j >= 0.88 and oj_i >= 0.88:
+                        drop_idx.add(i)
+                        break
+                    if _multiset_leq(ci, cj) and ni < nj and oi_j >= 0.9:
+                        drop_idx.add(i)
+                        break
+                    if _multiset_leq(cj, ci) and nj < ni and oj_i >= 0.9:
+                        drop_idx.add(j)
+                        continue
+
         if not drop_idx:
             return t
 
@@ -3675,6 +3758,136 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
     return out
 
 
+_RE_CENTER_DIV_TWO_DA3_TABLES = re.compile(
+    r'(<div\s+align=["\']center["\']\s*>\s*[^<]*?</div>\s*)'
+    r'(<table[^>]*>.*?</table>)'
+    r'(\s*<table[^>]*>.*?</table>)',
+    re.IGNORECASE | re.DOTALL,
+)
+
+
+def _da3_row_credit_vs_debit_category(desc: str) -> str:
+    """
+    Generic classification for Date|Description|Amount rows (middle column only).
+    Used to detect OCR pasting a withdrawals table after a deposits heading.
+    """
+    d = (desc or "").lower()
+    if "debit card return" in d:
+        return "credit"
+    if "debit card" in d and "return" in d:
+        return "credit"
+    if "deposit" in d and "debit card" not in d[:60]:
+        return "credit"
+    if "incoming wire" in d or "wire ref" in d:
+        return "credit"
+    if "zelle business reversal" in d or ("reversal" in d and "zelle" in d):
+        return "credit"
+    if "lrm claims" in d or ("claims" in d and "adj" in d):
+        return "credit"
+    # Credits / tax refunds that omit the word "deposit" (common on statement continuations)
+    if "adp tax" in d and "customer" in d:
+        return "credit"
+    if "debit card" in d:
+        return "debit"
+    if "ach corp debit" in d:
+        return "debit"
+    if "bus online ach settlement" in d:
+        return "debit"
+    if "zelle business payment to" in d:
+        return "debit"
+    if "visa money transfer debit" in d:
+        return "debit"
+    if "telephone payment" in d or "service charges" in d:
+        return "debit"
+    if "merch fee" in d:
+        return "debit"
+    if "deposit" in d:
+        return "credit"
+    return "unknown"
+
+
+def _da3_table_credit_debit_fractions(grid):
+    """Returns (debit_frac, credit_frac, unknown_frac, scored_row_count)."""
+    deb = cre = unk = 0
+    if not grid or len(grid) < 2:
+        return 0.0, 0.0, 0.0, 0
+    for row in grid[1:]:
+        cells = [str(c or "").strip() for c in row]
+        while len(cells) < 3:
+            cells.append("")
+        if not _DA3_LEADING_DATE.match(cells[0]):
+            continue
+        if not _DA3_STRICT_AMOUNT.match((cells[2] or "").strip()):
+            continue
+        cat = _da3_row_credit_vs_debit_category(cells[1])
+        if cat == "debit":
+            deb += 1
+        elif cat == "credit":
+            cre += 1
+        else:
+            unk += 1
+    tot = deb + cre + unk
+    if tot == 0:
+        return 0.0, 0.0, 0.0, 0
+    return deb / tot, cre / tot, unk / tot, tot
+
+
+def _drop_da3_misplaced_debit_after_deposit_heading_tables(text: str) -> str:
+    """
+    Fix 25: PDF order is withdrawals → then deposits. OCR sometimes emits two DA3 tables
+    after the deposits heading: real deposits, then a duplicate withdrawals block.
+    Drop the second table when it is overwhelmingly debit-like and the first is credit-like.
+    Repeats until no more matches (multiple duplicate blocks).
+    """
+    if not text or "<table" not in text.lower():
+        return text
+    if "deposit" not in text.lower():
+        return text
+
+    for _ in range(12):
+        replaced = False
+        pos = 0
+        while True:
+            m = _RE_CENTER_DIV_TWO_DA3_TABLES.search(text, pos)
+            if not m:
+                break
+            div = m.group(1)
+            t1 = m.group(2)
+            t2 = m.group(3)
+            sl = div.lower()
+            if "deposit" not in sl or not any(k in sl for k in ("credit", "interest", "credits")):
+                pos = m.start() + 1
+                continue
+
+            g1 = _parse_da3_grid_from_table_html(t1)
+            g2 = _parse_da3_grid_from_table_html(t2)
+            if not g1 or not g2 or not _is_da3_header(g1) or not _is_da3_header(g2):
+                pos = m.start() + 1
+                continue
+
+            db1, cr1, unk1, n1 = _da3_table_credit_debit_fractions(g1)
+            db2, cr2, _unk2, n2 = _da3_table_credit_debit_fractions(g2)
+            if n1 < 5 or n2 < 10:
+                pos = m.start() + 1
+                continue
+            # First table: deposits often have ADP/merchant lines scored as unknown — blend unk toward credit.
+            credit_like_first = cr1 + 0.42 * unk1
+            if credit_like_first < 0.30 and credit_like_first <= db1 + 0.05:
+                pos = m.start() + 1
+                continue
+            if db2 < 0.74 or db2 <= cr2 + 0.07:
+                pos = m.start() + 1
+                continue
+
+            text = text[: m.start()] + div + t1 + text[m.end() :]
+            replaced = True
+            break
+        if not replaced:
+            break
+
+    return text
+
+
 def stabilize_tables_and_text(page_md: str) -> str:
     if not page_md:
         return page_md
@@ -3697,6 +3910,8 @@ def stabilize_tables_and_text(page_md: str) -> str:
     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
     stabilized = _dedupe_subset_transaction_tables_html(stabilized)
+    stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
+    stabilized = _strip_orphan_continued_da3_flatline(stabilized)
     return close_unclosed_html(stabilized)
 
 # --------------------------
@@ -3820,6 +4035,7 @@ def run_ocr(uploaded_file):
                 # Safe no-op for scanned PDFs (no text layer) and non-transaction pages.
                 if is_pdf and not is_nfcu_doc:
                     stabilized = _patch_ocr_with_textlayer(stabilized, path, page_num)
+                    stabilized = _strip_orphan_continued_da3_flatline(stabilized)
                     # _patch_ocr_with_textlayer may append Case B rows from the PDF text layer
                     # with fused UCB-style cells; run the same HTML table pass again (Fix 13/14).
                     stabilized = normalize_html_tables(stabilized)
@@ -3827,6 +4043,7 @@ def run_ocr(uploaded_file):
                     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
                     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
                     stabilized = _dedupe_subset_transaction_tables_html(stabilized)
+                    stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
                 elif is_pdf and is_nfcu_doc:
                     # Navy-only additive fix: rebuild malformed summary table from
                     # text layer, but keep GLM body/header/footer unchanged.
@@ -3907,10 +4124,13 @@ def run_ocr(uploaded_file):
         merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
         # Cross-page: UCB section split + same transaction tables dedupe once on full doc.
         if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
+            merged = _strip_orphan_continued_da3_flatline(merged)
             merged = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(merged)
             merged = _split_ucb_deposits_electronic_sections_html(merged)
             merged = _patch_da3_msft_g_ref_amounts_across_tables(merged)
             merged = _dedupe_subset_transaction_tables_html(merged)
+            merged = _drop_da3_misplaced_debit_after_deposit_heading_tables(merged)
+            merged = _strip_orphan_continued_da3_flatline(merged)
         return merged
 
     except Exception as e: