Update app.py

app.py

@@ -53,8 +53,10 @@ Primary goals for reconcile rate:
             debits. Reorder to Credits + ITM table + Electronic Debits + merged debits; drop dup keys from the
             second table against the first; merge remaining orphan rows into the debits table.
 - Fix 17i: After Fix 17b splits ITM prefix from debits, OCR often leaves *credit* rows (POS Return, DDA
-            Transfer IN, Apple Cash INST R / CUPERTINO) at the *end* of the debits table — move that contiguous
-            tail from Electronic Debits back to Electronic Credits (same statement layout as the PDF).
+            Transfer IN, Apple Cash INST R / CUPERTINO) in the debits grid — move those rows to Electronic
+            Credits (contiguous-tail case on g_ed after 17b, or any matching row via 17j).
+- Fix 17j: When OCR emits a *second* Electronic Debits table (duplicate block), credit-only rows often appear
+            only there — relocate them to the nearest preceding Electronic Credits table (same patterns as 17i).
 - Fix 17e: Repeat EC/ED fused + 17d passes until stable (handles chained replacements). Centered section titles
             match both align=\"center\" and style=\"text-align:center\" on divs.
 - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
@@ -94,6 +96,9 @@ Primary goals for reconcile rate:
             relocated to just before the orphan table. Guards: no debit heading already before
             table; credit context must precede table; no other table between orphan and heading;
             try/except wrapper (safe no-op on any error).
+- Fix 31: On check / receipt image pages, GLM sometimes emits bogus table rows (e.g. \"Witness LA…\",
+            \"[hotline Control]\" after amounts, long trace ids). Strip those rows and bracket junk after
+            currency amounts; drop a stray \"## UCB\" heading before such tables. Safe no-op when absent.
 - Fix 20-override: _dedupe_duplicate_rows_in_da3_table now uses the PDF text-layer frequency
             map (_tl_freq_context) as its cap so legitimate repeated transactions (e.g. 6x
             Temporary Credit Reversal on the same date/amount) are preserved rather than
@@ -3021,18 +3026,20 @@ def _is_ucb_electronic_credit_row_after_split(grid, r: int) -> bool:
 
 
 def _ucb_move_trailing_credit_rows_from_ed_to_ec(g_ec, g_ed):
-    """Fix 17i: contiguous tail of credit-like rows at bottom of g_ed -> append to g_ec."""
+    """Fix 17i: credit-like rows that landed in g_ed (POS Return, DDA Transfer IN, Apple INST R) -> g_ec.
+
+    OCR often orders them after outgoing debits on the same date block, so a contiguous tail scan
+    misses them; we relocate every matching data row while preserving relative order.
+    """
     if not g_ec or not g_ed or len(g_ed) < 2 or not _is_da3_header(g_ec) or not _is_da3_header(g_ed):
         return g_ec, g_ed
-    i = len(g_ed) - 1
-    while i >= 1 and _is_ucb_electronic_credit_row_after_split(g_ed, i):
-        i -= 1
-    tail_start = i + 1
-    if tail_start >= len(g_ed):
+    credit_indices = [r for r in range(1, len(g_ed)) if _is_ucb_electronic_credit_row_after_split(g_ed, r)]
+    if not credit_indices:
         return g_ec, g_ed
-    credit_tail = [list(g_ed[r]) for r in range(tail_start, len(g_ed))]
-    g_ed_new = [list(g_ed[0])] + [list(g_ed[r]) for r in range(1, tail_start)]
-    g_ec_new = [list(g_ec[0])] + [list(r) for r in g_ec[1:]] + credit_tail
+    credit_rows = [list(g_ed[r]) for r in credit_indices]
+    drop = set(credit_indices)
+    g_ed_new = [list(g_ed[0])] + [list(g_ed[r]) for r in range(1, len(g_ed)) if r not in drop]
+    g_ec_new = [list(g_ec[0])] + [list(r) for r in g_ec[1:]] + credit_rows
     max_c = max(len(r) for r in g_ec_new + g_ed_new)
     for row in g_ec_new + g_ed_new:
         while len(row) < max_c:
@@ -3483,6 +3490,82 @@ def _ucb_prune_ed_fragment_dupes_vs_prev_table(text: str) -> str:
         return text
 
 
+_UCB_ED_DIV_AND_TABLE = re.compile(
+    _UCB_CENTER_DIV + r"\s*Electronic\s+Debits\s*</div>\s*(?P<tbl><table[^>]*>.*?</table>)",
+    re.DOTALL | re.IGNORECASE,
+)
+_UCB_EC_DIV_AND_TABLE = re.compile(
+    _UCB_CENTER_DIV + r"\s*Electronic\s+Credits\s*</div>\s*(?P<tbl><table[^>]*>.*?</table>)",
+    re.DOTALL | re.IGNORECASE,
+)
+
+
+def _ucb_relocate_credit_rows_from_ed_tables_to_ec_html(text: str) -> str:
+    """
+    Fix 17j: Credit-side rows (POS Return, DDA Transfer IN, Apple Cash INST R) sometimes appear only under a
+    duplicate Electronic Debits table, not in the fused EC table — move them to the preceding EC table.
+    """
+    if not text or "electronic debits" not in text.lower():
+        return text
+    try:
+        while True:
+            replaced = False
+            for ed_m in _UCB_ED_DIV_AND_TABLE.finditer(text):
+                g_ed = _parse_da3_grid_from_table_html(ed_m.group("tbl"))
+                if not g_ed or len(g_ed) < 2 or not _is_da3_header(g_ed):
+                    continue
+                credit_ix = [
+                    r
+                    for r in range(1, len(g_ed))
+                    if _is_ucb_electronic_credit_row_after_split(g_ed, r)
+                ]
+                if not credit_ix:
+                    continue
+                before = text[: ed_m.start()]
+                ec_matches = list(_UCB_EC_DIV_AND_TABLE.finditer(before))
+                if not ec_matches:
+                    continue
+                ec_m = ec_matches[-1]
+                g_ec = _parse_da3_grid_from_table_html(ec_m.group("tbl"))
+                if not g_ec or len(g_ec) < 2 or not _is_da3_header(g_ec):
+                    continue
+                drop = set(credit_ix)
+                credit_rows = [list(g_ed[r]) for r in credit_ix]
+                g_ed_new = [list(g_ed[0])] + [list(g_ed[r]) for r in range(1, len(g_ed)) if r not in drop]
+                g_ec_new = [list(g_ec[0])] + [list(r) for r in g_ec[1:]] + credit_rows
+                max_c = max(len(r) for r in g_ec_new + g_ed_new)
+                for row in g_ec_new + g_ed_new:
+                    while len(row) < max_c:
+                        row.append("")
+                new_ec = _grid_to_html(g_ec_new)
+                if len(g_ed_new) <= 1:
+                    text = (
+                        text[: ec_m.start("tbl")]
+                        + new_ec
+                        + text[ec_m.end("tbl") : ed_m.start()]
+                        + text[ed_m.end() :]
+                    )
+                else:
+                    new_ed = _grid_to_html(g_ed_new)
+                    text = (
+                        text[: ec_m.start("tbl")]
+                        + new_ec
+                        + text[ec_m.end("tbl") : ed_m.start()]
+                        + text[ed_m.start() : ed_m.start("tbl")]
+                        + new_ed
+                        + text[ed_m.end() :]
+                    )
+                log.info("Fix 17j: relocated %d credit row(s) from Electronic Debits to Electronic Credits", len(credit_rows))
+                replaced = True
+                break
+            if not replaced:
+                break
+        return text
+    except Exception as e:
+        log.warning("_ucb_relocate_credit_rows_from_ed_tables_to_ec_html failed: %s", e)
+        return text
+
+
 def _split_ucb_fused_ec_ed_headings_single_table_html(text: str) -> str:
     if not text or "electronic credits" not in text.lower():
         return text
@@ -3547,6 +3630,7 @@ def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
         nxt = _split_ucb_fused_ec_ed_headings_single_table_html(nxt)
         nxt = _ucb_strip_redundant_ed_subset_fragment(nxt)
         nxt = _ucb_prune_ed_fragment_dupes_vs_prev_table(nxt)
+        nxt = _ucb_relocate_credit_rows_from_ed_tables_to_ec_html(nxt)
         if nxt == text:
             break
         text = nxt
@@ -4265,6 +4349,20 @@ def _fix29_move_debit_heading_before_orphan_table(text: str) -> str:
             orphan_m = da3_matches[i]
             orphan_start = orphan_m.start()
             orphan_end = orphan_m.end()
+            orphan_tbl_html = orphan_m.group(0)
+
+            # Do not treat UCB Electronic Credits ITM-only tables as "orphan debits" needing a heading
+            # moved ahead of them — Fix 17b leaves ED after the ITM block; moving ED here stacks both
+            # headings above ITM and leaves debits under the wrong banner.
+            try:
+                og = _parse_da3_grid_from_table_html(orphan_tbl_html)
+                if og and (
+                    _ucb_table_credits_only_no_strong_debits(og)
+                    or _ucb_html_table_body_looks_itm_credits_only(orphan_tbl_html)
+                ):
+                    continue
+            except Exception:
+                pass
 
             before_window = text[max(0, orphan_start - _HEADING_SCAN_WINDOW): orphan_start]
             before_clean = re.sub(r"<[^>]+>", " ", before_window)
@@ -4326,6 +4424,73 @@ def _fix29_move_debit_heading_before_orphan_table(text: str) -> str:
         return text
 
 
+def _strip_check_image_ocr_junk_html(text: str) -> str:
+    """
+    Fix 31: Remove hallucinated rows on scanned check/receipt regions and normalize amount cells.
+    """
+    if not text or "<table" not in text.lower():
+        return text
+    try:
+        text = re.sub(
+            r"(\$[\d,]+\.\d{2})\s*\[[^\]]+\]",
+            r"\1",
+            text,
+        )
+        tbl_pat = re.compile(r"(<table[^>]*>)([\s\S]*?)(</table>)", re.IGNORECASE)
+
+        def _tr_effectively_empty(tr: str) -> bool:
+            core = re.sub(r"<[^>]+>", " ", tr)
+            return not core.strip()
+
+        def repl(m) -> str:
+            open_tag, body, close_tag = m.group(1), m.group(2), m.group(3)
+            low = body.lower()
+            if not any(
+                k in low
+                for k in (
+                    "hotline",
+                    "witness la",
+                    "itm checking",
+                    "care none",
+                    "processed ",
+                    "course dut",
+                    "n00590409",
+                )
+            ):
+                return m.group(0)
+            kept: List[str] = []
+            for trm in re.finditer(r"<tr\b[^>]*>[\s\S]*?</tr>", body, re.IGNORECASE):
+                tr = trm.group(0)
+                tl = tr.lower()
+                if _tr_effectively_empty(tr):
+                    continue
+                if "[hotline" in tl or "hotline control" in tl:
+                    continue
+                if "witness la" in tl and ("course" in tl or "dut" in tl):
+                    continue
+                if "itm checking witness" in tl or (
+                    "itm checking" in tl and "witness" in tl
+                ):
+                    continue
+                if "care none" in tl and "account" in tl:
+                    continue
+                if "processed " in tl and " @ " in tl and "items" in tl:
+                    continue
+                if re.search(r"\bN\d{15,}\b", tr):
+                    continue
+                kept.append(tr)
+            if not kept:
+                return m.group(0)
+            return open_tag + "".join(kept) + close_tag
+
+        text = tbl_pat.sub(repl, text)
+        text = re.sub(r"\n## UCB\s*\n+(?=<table)", "\n", text, flags=re.IGNORECASE)
+        return text
+    except Exception as e:
+        log.warning("_strip_check_image_ocr_junk_html failed: %s", e)
+        return text
+
+
 def stabilize_tables_and_text(page_md: str) -> str:
     if not page_md:
         return page_md
@@ -4359,6 +4524,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
     stabilized = _fix29_move_debit_heading_before_orphan_table(stabilized)  # Fix 29
     stabilized = _strip_orphan_continued_da3_flatline(stabilized)
     stabilized = _strip_standalone_continued_banner_line(stabilized)
+    stabilized = _strip_check_image_ocr_junk_html(stabilized)  # Fix 31
     return close_unclosed_html(stabilized)
 
 # --------------------------
@@ -4489,6 +4655,7 @@ def run_ocr(uploaded_file):
                     stabilized = _strip_credits_prefix_from_debit_da3_table(stabilized)  # Fix 28
                     stabilized = _fix29_move_debit_heading_before_orphan_table(stabilized)  # Fix 29
                     stabilized = _strip_standalone_continued_banner_line(stabilized)
+                    stabilized = _strip_check_image_ocr_junk_html(stabilized)  # Fix 31
                 elif is_pdf and is_nfcu_doc:
                     nfcu_summary = _extract_nfcu_summary_table(path, page_num)
                     if nfcu_summary:
@@ -4567,6 +4734,7 @@ def run_ocr(uploaded_file):
             merged = _fix29_move_debit_heading_before_orphan_table(merged)  # Fix 29
             merged = _strip_orphan_continued_da3_flatline(merged)
             merged = _strip_standalone_continued_banner_line(merged)
+            merged = _strip_check_image_ocr_junk_html(merged)  # Fix 31
         return merged
 
     except Exception as e: