Update app.py

app.py

@@ -36,11 +36,16 @@ Primary goals for reconcile rate:
             debit pattern) and at least one ITM DEPOSIT row exists, OCR stacked headings wrong — emit
             Electronic Credits, then the table, then Electronic Debits (section continues below). Safe no-op
             if the Debits heading was not before the table or any row looks like a card/ATM debit.
+            GUARD: Do NOT fire Fix 17c if ALL rows are ITM DEPOSIT — that indicates a Deposits (continued)
+            block that was incorrectly preceded by EC/ED headings, not a genuine Electronic Credits table.
 - Fix 17c-html: Same as 17c using raw HTML cues when TableGridParser/header normalization would skip the table.
+            Same ITM-only guard applied.
 - Fix 17d: Drop a second Electronic Debits block + DA3 table when its rows are a loose (date+amount)
             multiset subset of the immediately preceding DA3 table (OCR duplicate tail / wrong split).
+            GUARD: Do NOT drop a small table if it contains genuine credit-side rows (POS Return,
+            Acct Fund INST, DDA Transfer IN) — these are misplaced EC rows, not ED duplicates.
 - Fix 17e: Repeat EC/ED fused + 17d passes until stable (handles chained replacements). Centered section titles
-            match both align=\"center\" and style=\"text-align:center\" on divs.
+            match both align="center" and style="text-align:center" on divs.
 - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
             (later subset of earlier, earlier subset of later, or identical row sets)
 - Fix 18: adjacent DA3 tables — if the last K data rows of table N match the first K rows of table N+1
@@ -87,6 +92,9 @@ Primary goals for reconcile rate:
             earlier larger DA3 table — catches OCR-truncated description duplicates such as
             the BoA service-fee page where a VIDDYOZE -1.11 fragment table appears after the
             full service-fee table with a longer description string.
+- Fix 17c/17c-html/17d GUARD (UCB page 3): Prevent misclassification of "Deposits (continued)"
+            ITM DEPOSIT blocks as Electronic Credits, and prevent Fix 17d from dropping the
+            real Electronic Credits table (POS Return / Acct Fund INST / DDA Transfer IN).
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -1115,10 +1123,6 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
             key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
             tl_freq[key] = tl_freq.get(key, 0) + 1
 
-        # Share with _dedupe_duplicate_rows_in_da3_table (Fix-20 override).
-        # Keys MUST match _transaction_row_dedupe_key (date/desc/amount, no spaces in
-        # date/desc; amount stripped of $/,/space). The injection map tl_freq uses
-        # _norm() with spaces — different tuple shape — so we keep a parallel count.
         global _tl_freq_context
         tl_dedupe = {}
         for r in tl_rows:
@@ -2678,7 +2682,7 @@ def _transaction_row_dedupe_key(cells):
     date_s = re.sub(r"\s+", "", date_s)
     amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
     desc_s = html.unescape(desc_s)
-    desc_s = desc_s.replace("`", "'").replace("'", "'")
+    desc_s = desc_s.replace("`", "'").replace("\u2019", "'")
     desc_s = re.sub(r"\s+", "", desc_s.lower())
     return (date_s, desc_s, amt_s)
 
@@ -2747,11 +2751,6 @@ _DA3_MONEY_TOKEN = re.compile(
 )
 _MSFT_PIN_G_REF = re.compile(r"MICROSOFT#(G\d{6,16})", re.IGNORECASE)
 
-# Module-level text-layer frequency context (Fix-20 override).
-# Set by _patch_ocr_with_textlayer; consumed by normalize_html_tables in run_ocr's
-# post-patch pass. Cleared at the start of stabilize_tables_and_text (new page)
-# and on patch failure so keys never bleed across pages.
-# Keys match _transaction_row_dedupe_key (not the _norm()-based tl_freq map).
 _tl_freq_context: dict = {}
 
 _DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
@@ -2842,12 +2841,6 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
                 break
             j += 1
         run_len = j - i
-        # Cap at 2 by default (Fix 20: collapse OCR stutters) but honour
-        # the text-layer-confirmed count when available, so legitimate
-        # repeated bank transactions (e.g. 6x Temporary Credit Reversal
-        # on the same date/amount) are not incorrectly collapsed to 2.
-        # Keys absent from _tl_freq_context still cap at 2 — safe no-op
-        # for scanned PDFs or pages where no text layer was extracted.
         tl_confirmed = _tl_freq_context.get(k, 0)
         cap = max(tl_confirmed, 2) if tl_confirmed > 0 else 2
         take = min(run_len, cap)
@@ -2942,10 +2935,38 @@ def _ucb_has_itm_deposit_data_row(grid) -> bool:
     return False
 
 
+def _ucb_all_rows_are_itm_deposit(grid) -> bool:
+    """
+    Returns True if ALL data rows (row index >= 1) in a DA3 grid are
+    ITM DEPOSIT rows. Used to guard Fix 17c / 17c-html: a table that is
+    exclusively ITM DEPOSIT entries is a "Deposits (continued)" block,
+    NOT an Electronic Credits table, even when it has no strong-debit rows.
+    """
+    if not grid or len(grid) < 2:
+        return False
+    data_rows = [grid[r] for r in range(1, len(grid))
+                 if any(str(c or "").strip() for c in grid[r])]
+    if not data_rows:
+        return False
+    return all(_is_itm_deposit_row_ucb(grid, r) for r in range(1, len(grid))
+               if any(str(c or "").strip() for c in grid[r]))
+
+
 def _ucb_table_credits_only_no_strong_debits(grid) -> bool:
-    """True if DA3 table has real data rows but none match strong electronic-debit patterns."""
+    """
+    True if DA3 table has real data rows but none match strong
+    electronic-debit patterns AND at least one ITM DEPOSIT row exists.
+
+    FIX 17c GUARD: Returns False when ALL data rows are ITM DEPOSIT.
+    That case is a "Deposits (continued)" block — not Electronic Credits —
+    so Fix 17c must NOT reorder it under an EC heading.
+    """
     if not grid or len(grid) < 2 or not _is_da3_header(grid):
         return False
+    # Guard: if every row is ITM DEPOSIT, this is Deposits (continued),
+    # not Electronic Credits. Do not trigger Fix 17c.
+    if _ucb_all_rows_are_itm_deposit(grid):
+        return False
     has_data = False
     for r in range(1, len(grid)):
         row_txt = _row_text_full_ucb(grid, r).strip()
@@ -2961,6 +2982,9 @@ def _ucb_html_table_body_looks_itm_credits_only(table_html: str) -> bool:
     """
     Detect ITM-only credit block without relying on parsed grid headers (Fix 17c-html).
     Conservative: require ITM deposit text and absence of common electronic-debit verbs in the HTML.
+
+    FIX 17c-html GUARD: Returns False when the table contains ONLY ITM DEPOSIT rows
+    (no other description text), as that indicates a Deposits (continued) block.
     """
     if not table_html or "<table" not in table_html.lower():
         return False
@@ -2971,6 +2995,30 @@ def _ucb_html_table_body_looks_itm_credits_only(table_html: str) -> bool:
         return False
     if "recur payment" in low or "debit card" in low:
         return False
+    # Additional guard: if every <td> description cell only contains "ITM DEPOSIT"
+    # (no other transaction types), treat this as Deposits (continued), not EC.
+    # Extract description cell texts from <td> elements (skip header row <th>).
+    td_texts = re.findall(r"<td>([^<]*)</td>", table_html, re.IGNORECASE)
+    if td_texts:
+        # td_texts from a DA3 table come in triplets: date, desc, amount
+        # Check if any non-date, non-amount cell has text other than "ITM DEPOSIT"
+        non_itm_desc = False
+        for i, txt in enumerate(td_texts):
+            txt_stripped = txt.strip().lower()
+            if not txt_stripped:
+                continue
+            # Skip date-like tokens and amount-like tokens
+            if re.match(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$", txt_stripped):
+                continue
+            if re.match(r"^\$?[\d,]+\.\d{2}$", txt_stripped):
+                continue
+            # It's a description cell
+            if "itm deposit" not in txt_stripped and "itm dep" not in txt_stripped:
+                non_itm_desc = True
+                break
+        if not non_itm_desc:
+            # All description cells are ITM DEPOSIT — this is Deposits (continued)
+            return False
     return True
 
 
@@ -3028,6 +3076,7 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
 
     # Fix 17c: headings stacked as EC / ED / table but rows are all credits (e.g. ITM-only) —
     # put the table under Electronic Credits, then leave Electronic Debits for continuation.
+    # GUARD: Do NOT fire if all rows are ITM DEPOSIT (that is Deposits continued, not EC).
     if had_ed_heading_immediately_before_table and _ucb_table_credits_only_no_strong_debits(grid):
         sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
         parts = []
@@ -3043,6 +3092,7 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
         return "\n\n".join(parts)
 
     # Fix 17c-html: same reorder when grid header checks fail but raw HTML is clearly ITM-only credits.
+    # GUARD: _ucb_html_table_body_looks_itm_credits_only now returns False for pure ITM DEPOSIT tables.
     if had_ed_heading_immediately_before_table and _ucb_html_table_body_looks_itm_credits_only(table_html):
         sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
         parts = []
@@ -3069,8 +3119,6 @@ _UCB_CENTER_DIV = (
     r')'
 )
 
-# Optional empty centered banner only; EC/ED title divs are re-emitted after a successful split
-# so we do not duplicate headings.
 _UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
     r'(?P<lead>(?:' + _UCB_CENTER_DIV + r'\s*</div>\s*)?)'
     r'(?P<ecdiv>' + _UCB_CENTER_DIV + r'\s*Electronic\s+Credits\s*</div>\s*)'
@@ -3080,10 +3128,32 @@ _UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
 )
 
 
+def _ucb_table_html_has_genuine_credit_rows(table_html: str) -> bool:
+    """
+    Returns True if the table contains rows that are clearly credit-side
+    (POS Return, Acct Fund INST/CUPERTINO, DDA Transfer IN).
+    Used by Fix 17d guard to prevent dropping the real EC table.
+    """
+    if not table_html:
+        return False
+    low = table_html.lower()
+    if "pos return" in low:
+        return True
+    if "acct fund" in low and ("inst" in low or "cupertino" in low):
+        return True
+    if "dda transfer" in low and "in" in low:
+        return True
+    return False
+
+
 def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
     """
     Fix 17d: Remove a duplicate <div>Electronic Debits</div> + <table> when that table's rows are a
     loose (date, amount) multiset subset of the DA3 table that appears immediately before this block.
+
+    GUARD: Do NOT drop the table if it contains genuine credit-side rows (POS Return,
+    Acct Fund INST/CUPERTINO, DDA Transfer IN). Those rows belong under Electronic Credits
+    and must not be silently discarded as ED duplicates.
     """
     if not text or "electronic debits" not in text.lower():
         return text
@@ -3095,7 +3165,6 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
         )
         tbl_pat = re.compile(r'<table[^>]*>.*?</table>', re.DOTALL | re.IGNORECASE)
         _DATE_NONEMPTY = re.compile(r"^\d{1,2}[/\-]\d{2}")
-        # Stripped amounts may be "2194.19" (no comma) — must not use comma-group regex.
         _AMT_STRIPPED_OK = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
 
         def _loose_da_key_cells(cells):
@@ -3144,6 +3213,14 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
                 cb, cf = Counter(kb), Counter(kf)
                 if not all(cf[k] <= cb.get(k, 0) for k in cf):
                     continue
+                # GUARD: Do not drop a table that contains genuine credit-side rows.
+                # These rows belong under Electronic Credits, not as ED duplicates.
+                if _ucb_table_html_has_genuine_credit_rows(m.group("tbl")):
+                    log.info(
+                        "Fix 17d: skipping drop of table with genuine credit rows "
+                        "(POS Return / Acct Fund INST / DDA Transfer IN)"
+                    )
+                    continue
                 text = text[: m.start()] + text[m.end() :]
                 changed = True
                 break
@@ -3428,21 +3505,6 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
     """
     Fix 30: Drop small DA3 table fragments (1-5 data rows) that are a loose
     subset of an earlier larger DA3 table, matched by (date, amount) key alone.
-
-    This catches OCR-truncated duplicate rows where description text differs
-    between the fragment and the anchor table, preventing the exact-key dedup
-    in _dedupe_subset_transaction_tables_html from firing.
-
-    Example: BoA service-fee page — VIDDYOZE -1.11 appears in a 1-row
-    fragment table after a full 2-row service-fee table that has the same
-    row with a much longer description string.
-
-    Guards:
-      - Fragment must have 1-5 data rows.
-      - Anchor table must have more data rows than the fragment.
-      - Every (date, amount) pair in the fragment must be covered by the anchor.
-      - Date must be non-empty; amount must look like currency.
-      - Wrapped in try/except (safe no-op on any error).
     """
     if not text or "<table" not in text.lower():
         return text
@@ -3456,7 +3518,6 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
         _AMT_NONEMPTY  = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
 
         def _da_key(cells):
-            """Loose (date_norm, amount_norm) key — description ignored."""
             parts = [str(c or "").strip() for c in cells[:3]]
             while len(parts) < 3:
                 parts.append("")
@@ -3465,7 +3526,6 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
             amt_s  = amt_s.replace("\u2212", "-").replace("\u2013", "-")
             return (date_s, amt_s)
 
-        # Parse all DA3 tables and collect their (date, amount) key lists.
         da3_info = []
         for m in matches:
             g = _parse_da3_grid_from_table_html(m.group(0))
@@ -3499,8 +3559,6 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
                 if tj["n"] <= ti["n"]:
                     continue
                 big_keys = Counter(tj["keys"])
-                # Every (date, amount) pair in the fragment must be covered
-                # by the anchor table (multiset subset check).
                 if all(frag_keys[k] <= big_keys.get(k, 0) for k in frag_keys):
                     drop_set.add(ti["match"].start())
                     log.info(
@@ -3801,14 +3859,6 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
     """
     Fix 28: When OCR prepends the rows of a preceding credits/deposits DA3 table
     onto the top of a debits/withdrawals DA3 table, strip that prefix.
-    Safety guards (ALL must pass):
-      1. Both tables must be valid DA3 (Date|Description|Amount header).
-      2. Target (later) table must follow a debit/withdrawal heading.
-      3. Source (earlier) table must follow a credit/deposit heading.
-      4. Only the contiguous leading prefix of matching rows is removed.
-      5. At least 1 data row must remain in the target after stripping.
-      6. Strip count must be >= 1.
-      7. Entire function wrapped in try/except — returns input unchanged on error.
     """
     if not text or "<table" not in text.lower():
         return text
@@ -4000,8 +4050,6 @@ def stabilize_tables_and_text(page_md: str) -> str:
     if not page_md:
         return page_md
 
-    # Fresh page: drop any text-layer dedupe caps so the first normalize pass
-    # (before _patch_ocr_with_textlayer) does not see stale keys from a prior page.
     _tl_freq_context.clear()
 
     page_md = normalize_money_glyphs(page_md)
@@ -4119,7 +4167,6 @@ def run_ocr(uploaded_file):
 
         all_pages = []
         for page_num, page_result in enumerate(results):
-            # Header normalize runs before body patch; do not reuse prior page caps.
             _tl_freq_context.clear()
             page_md, regions = get_page_md_and_regions(page_result)
             img_h = page_heights[page_num] if page_num < len(page_heights) else 1000