Update app.py

app.py

@@ -36,6 +36,9 @@ Primary goals for reconcile rate:
             debit pattern) and at least one ITM DEPOSIT row exists, OCR stacked headings wrong — emit
             Electronic Credits, then the table, then Electronic Debits (section continues below). Safe no-op
             if the Debits heading was not before the table or any row looks like a card/ATM debit.
+- Fix 17c-html: Same as 17c using raw HTML cues when TableGridParser/header normalization would skip the table.
+- Fix 17d: Drop a second Electronic Debits block + DA3 table when its rows are a loose (date+amount)
+            multiset subset of the immediately preceding DA3 table (OCR duplicate tail / wrong split).
 - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
             (later subset of earlier, earlier subset of later, or identical row sets)
 - Fix 18: adjacent DA3 tables — if the last K data rows of table N match the first K rows of table N+1
@@ -2952,6 +2955,23 @@ def _ucb_table_credits_only_no_strong_debits(grid) -> bool:
     return bool(has_data and _ucb_has_itm_deposit_data_row(grid))
 
 
+def _ucb_html_table_body_looks_itm_credits_only(table_html: str) -> bool:
+    """
+    Detect ITM-only credit block without relying on parsed grid headers (Fix 17c-html).
+    Conservative: require ITM deposit text and absence of common electronic-debit verbs in the HTML.
+    """
+    if not table_html or "<table" not in table_html.lower():
+        return False
+    low = table_html.lower()
+    if "itm deposit" not in low and "itm dep" not in low:
+        return False
+    if "pos purchase" in low or "atm withdrawal" in low:
+        return False
+    if "recur payment" in low or "debit card" in low:
+        return False
+    return True
+
+
 def _ucb_split_grid_electronic_credits_debits_after_itm_prefix(grid):
     """
     Fix 17b: one fused DA3 table under Electronic Credits (+ optional empty EC/ED headings) where OCR
@@ -3006,10 +3026,7 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
 
     # Fix 17c: headings stacked as EC / ED / table but rows are all credits (e.g. ITM-only) —
     # put the table under Electronic Credits, then leave Electronic Debits for continuation.
-    if (
-        had_ed_heading_immediately_before_table
-        and _ucb_table_credits_only_no_strong_debits(grid)
-    ):
+    if had_ed_heading_immediately_before_table and _ucb_table_credits_only_no_strong_debits(grid):
         sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
         parts = []
         if lead_before_ec and lead_before_ec.strip():
@@ -3023,20 +3040,112 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
         )
         return "\n\n".join(parts)
 
+    # Fix 17c-html: same reorder when grid header checks fail but raw HTML is clearly ITM-only credits.
+    if had_ed_heading_immediately_before_table and _ucb_html_table_body_looks_itm_credits_only(table_html):
+        sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
+        parts = []
+        if lead_before_ec and lead_before_ec.strip():
+            parts.append(lead_before_ec.rstrip())
+        parts.extend(
+            [
+                sect.format(title="Electronic Credits"),
+                table_html.strip(),
+                sect.format(title="Electronic Debits"),
+            ]
+        )
+        return "\n\n".join(parts)
+
     return None
 
 
+_UCB_CENTER_DIV = r'<div[^>]*\balign\s*=\s*["\']center["\'][^>]*>'
+
 # Optional empty centered banner only; EC/ED title divs are re-emitted after a successful split
 # so we do not duplicate headings.
 _UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
-    r'(?P<lead>(?:<div\s+align=["\']center["\']\s*>\s*</div>\s*)?)'
-    r'<div\s+align=["\']center["\']\s*>\s*Electronic\s+Credits\s*</div>\s*'
-    r'(?P<ed_before>(?:<div\s+align=["\']center["\']\s*>\s*Electronic\s+Debits\s*</div>\s*)?)'
+    r'(?P<lead>(?:' + _UCB_CENTER_DIV + r'\s*</div>\s*)?)'
+    r'(?P<ecdiv>' + _UCB_CENTER_DIV + r'\s*Electronic\s+Credits\s*</div>\s*)'
+    r'(?P<ed_before>(?:' + _UCB_CENTER_DIV + r'\s*Electronic\s+Debits\s*</div>\s*)?)'
     r'(?P<table><table[^>]*>.*?</table>)',
     re.DOTALL | re.IGNORECASE,
 )
 
 
+def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
+    """
+    Fix 17d: Remove a duplicate <div>Electronic Debits</div> + <table> when that table's rows are a
+    loose (date, amount) multiset subset of the DA3 table that appears immediately before this block.
+    """
+    if not text or "electronic debits" not in text.lower():
+        return text
+    try:
+        ed_block = re.compile(
+            _UCB_CENTER_DIV + r'\s*Electronic\s+Debits\s*</div>\s*'
+            r'(?P<tbl><table[^>]*>.*?</table>)',
+            re.DOTALL | re.IGNORECASE,
+        )
+        tbl_pat = re.compile(r'<table[^>]*>.*?</table>', re.DOTALL | re.IGNORECASE)
+        _DATE_NONEMPTY = re.compile(r"^\d{1,2}[/\-]\d{2}")
+        # Stripped amounts may be "2194.19" (no comma) — must not use comma-group regex.
+        _AMT_STRIPPED_OK = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
+
+        def _loose_da_key_cells(cells):
+            parts = [str(c or "").strip() for c in cells[:3]]
+            while len(parts) < 3:
+                parts.append("")
+            date_s = re.sub(r"\s+", "", parts[0])
+            amt_s = re.sub(r"[\s$,]", "", parts[2]).lower()
+            amt_s = amt_s.replace("\u2212", "-").replace("\u2013", "-")
+            return (date_s, amt_s)
+
+        def _loose_keys_from_grid(g):
+            if not g or not _is_da3_header(g):
+                return None
+            keys = []
+            for row in g[1:]:
+                cells = [str(c or "").strip() for c in row]
+                if not any(cells):
+                    continue
+                k = _loose_da_key_cells(cells)
+                if (
+                    _DATE_NONEMPTY.match(k[0])
+                    and _AMT_STRIPPED_OK.match(k[1])
+                    and "." in k[1]
+                ):
+                    keys.append(k)
+            return keys
+
+        for _ in range(12):
+            changed = False
+            for m in ed_block.finditer(text):
+                before = text[: m.start()]
+                prev_tbl_m = None
+                for tm in tbl_pat.finditer(before):
+                    prev_tbl_m = tm
+                if prev_tbl_m is None:
+                    continue
+                g_big = _parse_da3_grid_from_table_html(prev_tbl_m.group(0))
+                g_frag = _parse_da3_grid_from_table_html(m.group("tbl"))
+                kb = _loose_keys_from_grid(g_big)
+                kf = _loose_keys_from_grid(g_frag)
+                if not kb or not kf:
+                    continue
+                if len(kf) > 24 or len(kf) >= len(kb):
+                    continue
+                cb, cf = Counter(kb), Counter(kf)
+                if not all(cf[k] <= cb.get(k, 0) for k in cf):
+                    continue
+                text = text[: m.start()] + text[m.end() :]
+                changed = True
+                break
+            if not changed:
+                break
+        return text
+    except Exception as e:
+        log.warning("_ucb_strip_redundant_ed_subset_fragment failed: %s", e)
+        return text
+
+
 def _split_ucb_fused_ec_ed_headings_single_table_html(text: str) -> str:
     if not text or "electronic credits" not in text.lower():
         return text
@@ -3095,7 +3204,8 @@ def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
             return new
 
         text = _UCB_DEP_CONT_BLOCK.sub(_repl, text)
-    return _split_ucb_fused_ec_ed_headings_single_table_html(text)
+    text = _split_ucb_fused_ec_ed_headings_single_table_html(text)
+    return _ucb_strip_redundant_ed_subset_fragment(text)
 
 
 def _parse_da3_grid_from_table_html(table_html: str):