Update app.py

app.py

@@ -25,6 +25,7 @@ Primary goals for reconcile rate:
             Beginning/Ending balance rows (UCB-style wide summary tables)
 - Fix 14: UCB 3-col tables where Beginning Balance amount is fused into description and Amount is $0.00
 - Fix 15: UCB (and similar) 3-col rows where Date+Description are fused in col1 and Amount sits in col2 with col3 empty
+- Fix 16: remove duplicate rows within a single Date|Description|Amount table; multi-pass table dedupe + final doc-wide dedupe
 - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
             (later subset of earlier, earlier subset of later, or identical row sets)
 """
@@ -3094,6 +3095,8 @@ def normalize_html_tables(text: str) -> str:
 
             # Fix 3: fused key-value rows in summary sections (no-space guard — safe on all PDFs)
             grid = _fix_fused_keyvalue_rows(grid)
+            # Fix 16: duplicate data rows inside same Date|Description|Amount table (OCR repeats).
+            grid = _dedupe_duplicate_rows_in_da3_table(grid)
             # Fix 12: split fused summary+daily-balance combined table, if detected.
             split_tables = _split_combined_summary_daily_balance_grid(grid)
             if split_tables:
@@ -3107,6 +3110,50 @@ def normalize_html_tables(text: str) -> str:
     out.append(text[last:])
     return "".join(out)
 
+
+def _transaction_row_dedupe_key(cells):
+    """Stable key for Date|Description|Amount rows (OCR variants, HTML entities)."""
+    parts = [str(c or "").strip() for c in cells[:3]]
+    while len(parts) < 3:
+        parts.append("")
+    date_s, desc_s, amt_s = parts[0], parts[1], parts[2]
+    date_s = re.sub(r"\s+", "", date_s)
+    amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
+    desc_s = html.unescape(desc_s)
+    desc_s = desc_s.replace("`", "'").replace("’", "'")
+    desc_s = re.sub(r"\s+", "", desc_s.lower())
+    return (date_s, desc_s, amt_s)
+
+
+def _is_da3_header(grid):
+    if not grid or len(grid[0]) < 3:
+        return False
+    h = [re.sub(r"\s+", " ", str(c or "").strip().lower()) for c in grid[0]]
+    if len(h) != 3:
+        return False
+    return h[0] == "date" and "description" in h[1] and h[2] == "amount"
+
+
+def _dedupe_duplicate_rows_in_da3_table(grid):
+    """Drop repeated data rows within one Date|Description|Amount table (same normalized key)."""
+    if not grid or len(grid) < 2:
+        return grid
+    if not _is_da3_header(grid):
+        return grid
+    seen = set()
+    out = [grid[0]]
+    for row in grid[1:]:
+        cells = [str(c or "").strip() for c in row]
+        if not any(cells):
+            continue
+        k = _transaction_row_dedupe_key(cells)
+        if k in seen:
+            continue
+        seen.add(k)
+        out.append(row)
+    return out if len(out) < len(grid) else grid
+
+
 def _dedupe_subset_transaction_tables_html(text: str) -> str:
     """
     Remove duplicate 3-column Date|Description|Amount tables:
@@ -3114,6 +3161,8 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
       - Earlier table is a strict subset of a later one → drop earlier (UCB-style
         "Deposits (continued)" preview before the same rows appear in the full list).
       - Identical row sets → drop the later table.
+      - Fuzzy overlap (≥92% of later rows match earlier) → drop later (Electronic Debits vs main).
+      - Runs multiple passes until stable.
 
     Row matching uses normalized keys so OCR variants (e.g. "0 27" vs "027", backticks)
     do not prevent duplicate tables from being detected.
@@ -3122,9 +3171,6 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
         return text
 
     pattern = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
-    matches = list(pattern.finditer(text))
-    if len(matches) < 2:
-        return text
 
     def _parse_grid(html: str):
         try:
@@ -3134,34 +3180,6 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
         except Exception:
             return None
 
-    grids = []
-    for m in matches:
-        grids.append((m, _parse_grid(m.group(0))))
-
-    def _header_matches(grid):
-        if not grid or len(grid) < 2:
-            return False
-        h = [re.sub(r"\s+", " ", str(c or "").strip().lower()) for c in grid[0]]
-        if len(h) != 3:
-            return False
-        d_ok = h[0] == "date"
-        desc_ok = "description" in h[1]
-        amt_ok = h[2] == "amount"
-        return d_ok and desc_ok and amt_ok
-
-    def _norm_row_key(cells):
-        """Stable key for same logical row despite minor OCR/HTML differences."""
-        parts = [str(c or "").strip() for c in cells[:3]]
-        while len(parts) < 3:
-            parts.append("")
-        date_s, desc_s, amt_s = parts[0], parts[1], parts[2]
-        date_s = re.sub(r"\s+", "", date_s)
-        amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
-        desc_s = html.unescape(desc_s)
-        desc_s = desc_s.replace("`", "'").replace("’", "'")
-        desc_s = re.sub(r"\s+", "", desc_s.lower())
-        return (date_s, desc_s, amt_s)
-
     def _row_set(g):
         s = set()
         for r in g[1:]:
@@ -3170,56 +3188,84 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
             cells = [str(c or "").strip() for c in r]
             if not any(cells):
                 continue
-            s.add(_norm_row_key(cells))
+            s.add(_transaction_row_dedupe_key(cells))
         return s
 
-    drop_idx = set()
-    for i in range(1, len(grids)):
-        if i in drop_idx:
-            continue
-        mi, gi = grids[i]
-        if not _header_matches(gi):
-            continue
-        si = _row_set(gi)
-        if not si:
-            continue
+    def _one_pass(t: str) -> str:
+        matches = list(pattern.finditer(t))
+        if len(matches) < 2:
+            return t
 
-        for j in range(i):
-            if j in drop_idx:
+        grids = []
+        for m in matches:
+            grids.append((m, _parse_grid(m.group(0))))
+
+        drop_idx = set()
+        for i in range(1, len(grids)):
+            if i in drop_idx:
                 continue
-            mj, gj = grids[j]
-            if not _header_matches(gj):
+            mi, gi = grids[i]
+            if not _is_da3_header(gi):
                 continue
-            sj = _row_set(gj)
-            if not sj:
+            si = _row_set(gi)
+            if not si:
                 continue
 
-            # Earlier j fully contained in later i (preview + full list) → drop earlier.
-            if sj <= si and len(sj) < len(si):
-                drop_idx.add(j)
-                continue
+            for j in range(i):
+                if j in drop_idx:
+                    continue
+                mj, gj = grids[j]
+                if not _is_da3_header(gj):
+                    continue
+                sj = _row_set(gj)
+                if not sj:
+                    continue
 
-            # Later i strict subset of earlier j → drop later.
-            if si <= sj and len(si) < len(sj):
-                drop_idx.add(i)
-                break
+                # Earlier j fully contained in later i (preview + full list) → drop earlier.
+                if sj <= si and len(sj) < len(si):
+                    drop_idx.add(j)
+                    continue
 
-            if si == sj:
-                drop_idx.add(i)
-                break
+                # Later i strict subset of earlier j → drop later (e.g. Electronic Debits).
+                if si <= sj and len(si) < len(sj):
+                    drop_idx.add(i)
+                    break
 
-    if not drop_idx:
-        return text
+                if si == sj:
+                    drop_idx.add(i)
+                    break
 
-    out = []
-    last = 0
-    for idx, m in enumerate(matches):
-        out.append(text[last : m.start()])
-        if idx not in drop_idx:
-            out.append(m.group(0))
-        last = m.end()
-    out.append(text[last:])
-    return "".join(out)
+                inter = si & sj
+                # Later table's rows almost all appear in earlier table (OCR drift).
+                ri = len(inter) / len(si) if si else 0.0
+                rj = len(inter) / len(sj) if sj else 0.0
+                if ri >= 0.88 and len(si) <= len(sj):
+                    drop_idx.add(i)
+                    break
+                if rj >= 0.88 and len(sj) < len(si):
+                    drop_idx.add(j)
+                    continue
+
+        if not drop_idx:
+            return t
+
+        out = []
+        last = 0
+        for idx, m in enumerate(matches):
+            out.append(t[last : m.start()])
+            if idx not in drop_idx:
+                out.append(m.group(0))
+            last = m.end()
+        out.append(t[last:])
+        return "".join(out)
+
+    out = text
+    for _ in range(12):
+        nxt = _one_pass(out)
+        if nxt == out:
+            break
+        out = nxt
+    return out
 
 
 def stabilize_tables_and_text(page_md: str) -> str:
@@ -3444,7 +3490,11 @@ def run_ocr(uploaded_file):
             if parts:
                 all_pages.append("\n\n".join(parts))
 
-        return "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
+        merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
+        # Cross-page: same transaction tables split across page boundaries must dedupe once.
+        if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
+            merged = _dedupe_subset_transaction_tables_html(merged)
+        return merged
 
     except Exception as e:
         import traceback