Update app.py

app.py

@@ -58,13 +58,22 @@ Primary goals for reconcile rate:
 - Fix 28: Strip leading credit/deposit rows that OCR prepends to a withdrawals/debits DA3 table
             when the preceding DA3 table was under a deposits/credits heading. Only strips the
             contiguous prefix of the debit table whose rows appear in the credit table; requires
-            ≥1 data row to remain; entire function wrapped in try/except (safe no-op on any error).
+            >=1 data row to remain; entire function wrapped in try/except (safe no-op on any error).
 - Fix 29: Move a misplaced debit/withdrawal heading to before its orphan DA3 table when OCR
             places the heading AFTER the table instead of before it. Pattern detected:
             [credit heading+table] ... [orphan DA3 table] ... [debit heading] — the heading is
             relocated to just before the orphan table. Guards: no debit heading already before
             table; credit context must precede table; no other table between orphan and heading;
             try/except wrapper (safe no-op on any error).
+- Fix 20-override: _dedupe_duplicate_rows_in_da3_table now uses the PDF text-layer frequency
+            map (_tl_freq_context) as its cap so legitimate repeated transactions (e.g. 6x
+            Temporary Credit Reversal on the same date/amount) are preserved rather than
+            being collapsed to 2. Keys absent from the text layer still cap at 2 (OCR stutter
+            behaviour unchanged). Safe no-op for scanned PDFs with no text layer.
+- Fix 30: Drop small DA3 fragments (1-5 rows) that are a loose (date+amount) subset of an
+            earlier larger DA3 table — catches OCR-truncated description duplicates such as
+            the BoA service-fee page where a VIDDYOZE -1.11 fragment table appears after the
+            full service-fee table with a longer description string.
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -432,10 +441,6 @@ def _is_amount_like(s: str) -> bool:
     return re.fullmatch(r"\$?-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", s.strip()) is not None
 
 def _drop_truly_empty_columns(grid):
-    """
-    Drop columns that are empty in header AND almost always empty in body.
-    This fixes summary tables that have an extra blank trailing column.
-    """
     if not grid or len(grid) < 1:
         return grid
 
@@ -470,12 +475,6 @@ def _drop_truly_empty_columns(grid):
     return new_grid
 
 def _merge_blank_header_text_columns(grid):
-    """
-    If the header row has blank columns, and most body rows have non-empty *text* in that
-    blank column, merge that column into the nearest non-empty header to the left (usually
-    DESCRIPTION), then remove the blank column.
-    This fixes transaction tables where DESCRIPTION is split across two columns.
-    """
     if not grid or len(grid) < 2:
         return grid
 
@@ -536,16 +535,12 @@ def _merge_blank_header_text_columns(grid):
 # Shared keyword definitions
 # --------------------------
 
-# Matches a standalone money value across currencies.
-# Covers: $3,447.10  -1,234.56  £500.00  Rs1000  etc.
 _MONEY_RE = re.compile(
     r"^(?:[\$£€¥]|Rs\.?|INR|PKR)?\s*-?\s?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$",
     re.IGNORECASE,
 )
 
-# All column-header keywords recognised across any bank statement format.
 _HEADER_KW_PATTERNS = [
-    # Date variants
     r"DATE",
     r"POSTING\s+DATE",
     r"VALUE\s+DATE",
@@ -553,7 +548,6 @@ _HEADER_KW_PATTERNS = [
     r"TRANSACTION\s+DATE",
     r"ENTRY\s+DATE",
     r"EFFECTIVE\s+DATE",
-    # Transaction ID / reference
     r"TRANSACTION(?:\s+(?:ID|TYPE|NO|NUMBER))?",
     r"TXN(?:\s+(?:ID|NO|TYPE))?",
     r"REF(?:ERENCE)?(?:\s*(?:NO|NUM|NUMBER))?",
@@ -562,30 +556,25 @@ _HEADER_KW_PATTERNS = [
     r"VOUCHER(?:\s*(?:NO|NUMBER))?",
     r"SR\.?\s*NO\.?",
     r"SERIAL(?:\s*(?:NO|NUMBER))?",
-    # Description variants
     r"DESCRIPTION",
     r"DETAILS?",
     r"PARTICULARS?(?:\s+OF\s+TRANSACTION)?",
     r"NARRATION",
     r"REMARKS?",
     r"NOTES?",
-    # Debit variants
     r"DEBIT",
     r"DEBITS?",
     r"DR\.?",
     r"WITHDRAWALS?",
     r"PAID\s+OUT",
     r"MONEY\s+OUT",
-    # Credit variants
     r"CREDIT",
     r"CREDITS?",
     r"CR\.?",
     r"DEPOSITS?",
     r"PAID\s+IN",
     r"MONEY\s+IN",
-    # Amount
     r"AMOUNT",
-    # Balance variants
     r"BALANCE",
     r"BAL\.?",
     r"RUNNING\s+BALANCE",
@@ -601,20 +590,16 @@ _HEADER_KW_PATTERNS = [
     r"OPENING\s+BAL\.?",
 ]
 
-# Pre-compiled: each pattern anchored at start, case-insensitive
 _HEADER_KW_RES = [
     re.compile(r"^(" + p + r")(.*)", re.IGNORECASE | re.DOTALL)
     for p in _HEADER_KW_PATTERNS
 ]
 
-# Quick exact-match check: "is this entire string a known keyword?"
 _HEADER_KW_EXACT_RE = re.compile(
     r"^(?:" + r"|".join(_HEADER_KW_PATTERNS) + r")$",
     re.IGNORECASE,
 )
 
-# Minimum fraction of cells in a row that must be pure keywords for that row
-# to be considered a misplaced header row.
 _HEADER_ROW_KEYWORD_THRESHOLD = 0.5
 
 # --------------------------
@@ -1117,6 +1102,11 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
             key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
             tl_freq[key] = tl_freq.get(key, 0) + 1
 
+        # Share with _dedupe_duplicate_rows_in_da3_table so it can use
+        # the confirmed count as the cap (Fix-20 override).
+        global _tl_freq_context
+        _tl_freq_context = dict(tl_freq)
+
         table_pattern = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
         result = page_md
 
@@ -1291,6 +1281,9 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
     except Exception as e:
         log.warning("_patch_ocr_with_textlayer failed (page %d): %s", page_num, e)
         return page_md
+    finally:
+        # Always clear so stale frequencies do not affect subsequent pages.
+        _tl_freq_context.clear()
 
 # --------------------------
 # Fix 5: Mid-table separator row reconstruction
@@ -2736,6 +2729,15 @@ _DA3_MONEY_TOKEN = re.compile(
     r"(?<![\d,])(-?\$?\d{1,3}(?:,\d{3})*\.\d{2}|-?\$?\d{2,6}\.\d{2})(?!\d)"
 )
 _MSFT_PIN_G_REF = re.compile(r"MICROSOFT#(G\d{6,16})", re.IGNORECASE)
+
+# Module-level text-layer frequency context (Fix-20 override).
+# Set by _patch_ocr_with_textlayer before post-patch normalize passes;
+# cleared in a finally clause so it never bleeds across pages.
+# Keys are (date_norm, desc_norm, amount_norm) tuples matching tl_freq.
+# When a key is present, _dedupe_duplicate_rows_in_da3_table uses its value
+# as the cap instead of the default 2, preserving all text-layer-confirmed rows.
+_tl_freq_context: dict = {}
+
 _DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
 
 
@@ -2824,7 +2826,15 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
                 break
             j += 1
         run_len = j - i
-        take = min(run_len, 2)
+        # Cap at 2 by default (Fix 20: collapse OCR stutters) but honour
+        # the text-layer-confirmed count when available, so legitimate
+        # repeated bank transactions (e.g. 6x Temporary Credit Reversal
+        # on the same date/amount) are not incorrectly collapsed to 2.
+        # Keys absent from _tl_freq_context still cap at 2 — safe no-op
+        # for scanned PDFs or pages where no text layer was extracted.
+        tl_confirmed = _tl_freq_context.get(k, 0)
+        cap = max(tl_confirmed, 2) if tl_confirmed > 0 else 2
+        take = min(run_len, cap)
         for t in range(take):
             out.append(data_rows[i + t])
         i = j
@@ -3128,6 +3138,110 @@ def _trim_adjacent_da3_suffix_duplicating_next_table_prefix(text: str) -> str:
     return text
 
 
+def _dedupe_da3_by_date_amount_loose(text: str) -> str:
+    """
+    Fix 30: Drop small DA3 table fragments (1-5 data rows) that are a loose
+    subset of an earlier larger DA3 table, matched by (date, amount) key alone.
+
+    This catches OCR-truncated duplicate rows where description text differs
+    between the fragment and the anchor table, preventing the exact-key dedup
+    in _dedupe_subset_transaction_tables_html from firing.
+
+    Example: BoA service-fee page — VIDDYOZE -1.11 appears in a 1-row
+    fragment table after a full 2-row service-fee table that has the same
+    row with a much longer description string.
+
+    Guards:
+      - Fragment must have 1-5 data rows.
+      - Anchor table must have more data rows than the fragment.
+      - Every (date, amount) pair in the fragment must be covered by the anchor.
+      - Date must be non-empty; amount must look like currency.
+      - Wrapped in try/except (safe no-op on any error).
+    """
+    if not text or "<table" not in text.lower():
+        return text
+    try:
+        pattern = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
+        matches = list(pattern.finditer(text))
+        if len(matches) < 2:
+            return text
+
+        _DATE_NONEMPTY = re.compile(r"^\d{1,2}[/\-]\d{2}")
+        _AMT_NONEMPTY  = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
+
+        def _da_key(cells):
+            """Loose (date_norm, amount_norm) key — description ignored."""
+            parts = [str(c or "").strip() for c in cells[:3]]
+            while len(parts) < 3:
+                parts.append("")
+            date_s = re.sub(r"\s+", "", parts[0])
+            amt_s  = re.sub(r"[\s$,]", "", parts[2]).lower()
+            amt_s  = amt_s.replace("\u2212", "-").replace("\u2013", "-")
+            return (date_s, amt_s)
+
+        # Parse all DA3 tables and collect their (date, amount) key lists.
+        da3_info = []
+        for m in matches:
+            g = _parse_da3_grid_from_table_html(m.group(0))
+            if not g or not _is_da3_header(g) or len(g) < 2:
+                continue
+            keys = []
+            for row in g[1:]:
+                cells = [str(c or "").strip() for c in row]
+                if not any(cells):
+                    continue
+                k = _da_key(cells)
+                if _DATE_NONEMPTY.match(k[0]) and _AMT_NONEMPTY.match(k[1]):
+                    keys.append(k)
+            da3_info.append({"match": m, "keys": keys, "n": len(keys)})
+
+        if len(da3_info) < 2:
+            return text
+
+        drop_set = set()
+        for i in range(1, len(da3_info)):
+            ti = da3_info[i]
+            if ti["n"] == 0 or ti["n"] > 5:
+                continue
+            if ti["match"].start() in drop_set:
+                continue
+            frag_keys = Counter(ti["keys"])
+            for j in range(i - 1, -1, -1):
+                tj = da3_info[j]
+                if tj["match"].start() in drop_set:
+                    continue
+                if tj["n"] <= ti["n"]:
+                    continue
+                big_keys = Counter(tj["keys"])
+                # Every (date, amount) pair in the fragment must be covered
+                # by the anchor table (multiset subset check).
+                if all(frag_keys[k] <= big_keys.get(k, 0) for k in frag_keys):
+                    drop_set.add(ti["match"].start())
+                    log.info(
+                        "Fix 30: dropped DA3 fragment (%d rows) as loose "
+                        "(date+amount) subset of larger table (%d rows)",
+                        ti["n"], tj["n"],
+                    )
+                    break
+
+        if not drop_set:
+            return text
+
+        out = []
+        last = 0
+        for m in matches:
+            out.append(text[last: m.start()])
+            if m.start() not in drop_set:
+                out.append(m.group(0))
+            last = m.end()
+        out.append(text[last:])
+        return "".join(out)
+
+    except Exception as e:
+        log.warning("_dedupe_da3_by_date_amount_loose failed: %s", e)
+        return text
+
+
 def _dedupe_subset_transaction_tables_html(text: str) -> str:
     if not text or "<table" not in text.lower():
         return text
@@ -3374,28 +3488,20 @@ def _drop_da3_misplaced_debit_after_deposit_heading_tables(text: str) -> str:
 # Fix 28: Strip credit-row prefix from debit DA3 table
 # --------------------------
 
-# Keywords that identify a section heading as debit/withdrawal-oriented.
 _DEBIT_HEADING_KEYWORDS = re.compile(
     r"\b(withdrawal|withdrawals|debit|debits|charges?|subtraction|subtractions|"
     r"payments?\s+made|money\s+out|paid\s+out|electronic\s+debit)\b",
     re.IGNORECASE,
 )
-# Keywords that identify a section heading as credit/deposit-oriented.
 _CREDIT_HEADING_KEYWORDS = re.compile(
     r"\b(deposit|deposits|credit|credits|addition|additions|interest\s+paid|"
     r"money\s+in|paid\s+in|incoming|received)\b",
     re.IGNORECASE,
 )
-# How many characters before a table start to scan for a section heading.
 _HEADING_SCAN_WINDOW = 600
 
 
 def _get_heading_before_table(text: str, table_start: int) -> str:
-    """
-    Return the last non-trivial text line in the _HEADING_SCAN_WINDOW chars
-    that precede `table_start`.  HTML tags are stripped before scanning so that
-    centered <div> headings are picked up correctly.
-    """
     window = text[max(0, table_start - _HEADING_SCAN_WINDOW): table_start]
     clean = re.sub(r"<[^>]+>", " ", window)
     lines = [ln.strip() for ln in clean.splitlines() if ln.strip()]
@@ -3426,7 +3532,6 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
         if len(matches) < 2:
             return text
 
-        # Collect all DA3 tables with their heading context.
         da3_tables = []
         for m in matches:
             g = _parse_da3_grid_from_table_html(m.group(0))
@@ -3446,10 +3551,7 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
                     keys.add(_transaction_row_dedupe_key(cells))
             return keys
 
-        # For each DA3 table preceded by a debit heading, look backward for the
-        # nearest DA3 table preceded by a credit heading, and measure how many
-        # of the debit table's leading rows appear in that credit table.
-        drop_map = {}  # table match start -> number of rows to strip
+        drop_map = {}
         for i in range(1, len(da3_tables)):
             ti = da3_tables[i]
             hi = ti["heading"].lower()
@@ -3468,7 +3570,6 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
                 keys_j = _row_keys(gj)
                 if not keys_j:
                     continue
-                # Count the contiguous leading rows of data_i that are in keys_j.
                 strip_count = 0
                 for row in data_i:
                     cells = [str(c or "").strip() for c in row]
@@ -3479,7 +3580,6 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
                         break
                 if strip_count < 1:
                     continue
-                # Guard: at least 1 data row must survive.
                 if len(data_i) - strip_count < 1:
                     continue
                 drop_map[ti["match"].start()] = strip_count
@@ -3487,12 +3587,11 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
                     "Fix 28: stripped %d leading credit row(s) from debit table at offset %d",
                     strip_count, ti["match"].start(),
                 )
-                break  # found nearest credit table — stop looking further back
+                break
 
         if not drop_map:
             return text
 
-        # Apply the strip.
         out_chunks = []
         last = 0
         for m in matches:
@@ -3520,32 +3619,17 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
 # Fix 29: Move misplaced debit heading to before its orphan DA3 table
 # --------------------------
 
-# Centered div heading pattern (BoA / generic style)
 _CENTER_DIV_HDR_RE = re.compile(
     r'<div\s+align=["\']center["\']\s*>\s*(.*?)\s*</div>',
     re.IGNORECASE | re.DOTALL,
 )
-_HEADING_SCAN_FORWARD = 800   # chars after table end to search for misplaced heading
+_HEADING_SCAN_FORWARD = 800
 
 
 def _fix29_move_debit_heading_before_orphan_table(text: str) -> str:
     """
     Fix 29: When OCR places a debit/withdrawal section heading AFTER the withdrawals table
     instead of before it, move the heading to its correct position (just before the table).
-
-    Pattern detected:
-      [credit/deposit DA3 table]  ...no debit heading...  [orphan DA3 table]  [debit heading]
-      where no other DA3 table separates the orphan table from the debit heading.
-
-    Action: move the debit heading to just before the orphan DA3 table.
-
-    Safety guards (ALL must pass):
-      1. Orphan table must be a valid DA3 (Date|Description|Amount).
-      2. A debit heading must appear AFTER the orphan table within _HEADING_SCAN_FORWARD chars.
-      3. No debit heading appears immediately before the orphan table (within _HEADING_SCAN_WINDOW).
-      4. A credit/deposit heading must exist before the orphan table (within 1200 chars).
-      5. No other table appears between the orphan table end and the debit heading.
-      6. Entire function wrapped in try/except -- returns input unchanged on any error.
     """
     if not text or "<table" not in text.lower():
         return text
@@ -3566,7 +3650,6 @@ def _fix29_move_debit_heading_before_orphan_table(text: str) -> str:
             orphan_start = orphan_m.start()
             orphan_end = orphan_m.end()
 
-            # Guard 3: no debit heading already before the orphan table
             before_window = text[max(0, orphan_start - _HEADING_SCAN_WINDOW): orphan_start]
             before_clean = re.sub(r"<[^>]+>", " ", before_window)
             lines_before = [l.strip() for l in before_clean.splitlines() if l.strip()]
@@ -3576,24 +3659,20 @@ def _fix29_move_debit_heading_before_orphan_table(text: str) -> str:
                     last_meaningful = l
                     break
             if _DEBIT_HEADING_KEYWORDS.search(last_meaningful):
-                continue  # already has debit heading before it
+                continue
 
-            # Guard 4: credit context must precede the orphan table
             big_before = text[max(0, orphan_start - 1200): orphan_start]
             if not _CREDIT_HEADING_KEYWORDS.search(re.sub(r"<[^>]+>", " ", big_before)):
                 continue
 
-            # Guard 5: no other table between orphan end and the potential debit heading
             next_tbl_m = tbl_pat.search(text, orphan_end)
 
-            # Guard 2: find debit heading after the orphan table (within forward window)
             dh_start = dh_end = None
             for div_m in _CENTER_DIV_HDR_RE.finditer(text, orphan_end, orphan_end + _HEADING_SCAN_FORWARD):
                 div_clean = re.sub(r"<[^>]+>", "", div_m.group(1)).strip()
                 if _DEBIT_HEADING_KEYWORDS.search(div_clean) and not _CREDIT_HEADING_KEYWORDS.search(div_clean):
-                    # Guard 5: no table should be between orphan_end and this heading
                     if next_tbl_m and next_tbl_m.start() < div_m.start():
-                        break  # another table intervenes
+                        break
                     dh_start = div_m.start()
                     dh_end = div_m.end()
                     break
@@ -3606,7 +3685,6 @@ def _fix29_move_debit_heading_before_orphan_table(text: str) -> str:
         if not moves:
             return text
 
-        # Apply moves in reverse order to preserve offsets
         for orphan_start, dh_start, dh_end in reversed(moves):
             orphan_m2 = next((m for m in tbl_pat.finditer(text) if m.start() == orphan_start), None)
             if not orphan_m2:
@@ -3655,6 +3733,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
     stabilized = _dedupe_subset_transaction_tables_html(stabilized)
+    stabilized = _dedupe_da3_by_date_amount_loose(stabilized)  # Fix 30
     stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
     stabilized = _strip_credits_prefix_from_debit_da3_table(stabilized)  # Fix 28
     stabilized = _fix29_move_debit_heading_before_orphan_table(stabilized)  # Fix 29
@@ -3783,6 +3862,7 @@ def run_ocr(uploaded_file):
                     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
                     stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
                     stabilized = _dedupe_subset_transaction_tables_html(stabilized)
+                    stabilized = _dedupe_da3_by_date_amount_loose(stabilized)  # Fix 30
                     stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
                     stabilized = _strip_credits_prefix_from_debit_da3_table(stabilized)  # Fix 28
                     stabilized = _fix29_move_debit_heading_before_orphan_table(stabilized)  # Fix 29
@@ -3859,6 +3939,7 @@ def run_ocr(uploaded_file):
             merged = _split_ucb_deposits_electronic_sections_html(merged)
             merged = _patch_da3_msft_g_ref_amounts_across_tables(merged)
             merged = _dedupe_subset_transaction_tables_html(merged)
+            merged = _dedupe_da3_by_date_amount_loose(merged)  # Fix 30
             merged = _drop_da3_misplaced_debit_after_deposit_heading_tables(merged)
             merged = _strip_credits_prefix_from_debit_da3_table(merged)  # Fix 28
             merged = _fix29_move_debit_heading_before_orphan_table(merged)  # Fix 29