Update app.py

app.py

@@ -17,11 +17,13 @@ Primary goals for reconcile rate:
   8) reconstruct mid-table separator rows split across columns by OCR (e.g. 'Card account # XXXX 2 | | 889')
 - footer extraction enabled on all pages with deduplication guard (no double-print if body OCR already captured it)
 - Fix 4: cross-validate OCR table row counts against PDF text layer; inject missing duplicate rows
-         (safe no-op for scanned PDFs and non-transaction pages)
+         (safe no-op for scanned PDFs and non-transaction pages); after injection, re-run normalize_html_tables
+         + subset dedupe so UCB-style fused cells from the text layer get Fix 13/14
 - Fix NF-1: extract Navy Federal "Summary of your deposit accounts" table from PDF text layer
             using spatial word positions (GLM-OCR fails on this wide-column layout)
 - Fix 13: collapse Date|Description|Amount|Description|Amount tables to 3 columns; fix fused
             Beginning/Ending balance rows (UCB-style wide summary tables)
+- Fix 14: UCB 3-col tables where Beginning Balance amount is fused into description and Amount is $0.00
 - Post-pass: drop later duplicate subset Date|Description|Amount tables (repeated deposit blocks)
 """
 
@@ -2702,32 +2704,46 @@ def _normalize_double_desc_amount_header_table(grid):
     """
     Fix 13: Collapse OCR tables with a duplicated column group:
       Date | Description | Amount | Description | Amount
-    Common on United Community Bank-style statements where a second
-    Description/Amount pair holds sub-totals (e.g. Average Ledger Balance).
+    (or 4 cols if trailing Amount column was dropped). UCB-style summary tables
+    put labels like "9 Credit(s) This Period" in the Date column.
     Output: Date | Description | Amount
-
-    Strict guard: header row must match exactly those five labels (case/spacing
-    normalized). No bank name checks.
     """
     if not grid or len(grid) < 2:
         return grid
 
-    raw_h = [str(c or "").strip() for c in grid[0]]
-    header_norm = [re.sub(r"\s+", " ", h).lower() for h in raw_h]
-    if len(header_norm) != 5:
-        return grid
-    if header_norm != ["date", "description", "amount", "description", "amount"]:
+    def _hdr_tight(s):
+        return re.sub(r"\s+", "", str(s or "").strip().lower())
+
+    # OCR often inserts spaces inside words; compare whitespace-stripped tokens.
+    expected5_t = ["date", "description", "amount", "description", "amount"]
+    expected4_t = ["date", "description", "amount", "description"]
+
+    def _row_is_ucb_header(cells):
+        if len(cells) == 5:
+            return [_hdr_tight(c) for c in cells] == expected5_t
+        if len(cells) == 4:
+            return [_hdr_tight(c) for c in cells] == expected4_t
+        return False
+
+    hdr_idx = None
+    ncols = 0
+    for i in range(min(5, len(grid))):
+        raw_try = [str(c or "").strip() for c in grid[i]]
+        if _row_is_ucb_header(raw_try):
+            hdr_idx = i
+            ncols = len(raw_try)
+            break
+    if hdr_idx is None:
         return grid
 
-    date_only_re = re.compile(r"^\d{1,2}/\d{1,2}/\d{4}$")
-    date_prefix_re = re.compile(r"^(\d{1,2}/\d{1,2}/\d{4})\s*(.*)$")
+    date_only_re = re.compile(r"^\d{1,2}/\d{1,2}/\d{2,4}$")
+    date_prefix_re = re.compile(r"^(\d{1,2}/\d{1,2}/\d{2,4})\s*(.*)$")
     money_cell_re = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
 
     def _is_money(s):
         if not s or not str(s).strip():
             return False
-        t = str(s).strip()
-        return bool(money_cell_re.match(t))
+        return bool(money_cell_re.match(str(s).strip()))
 
     def _extract_money(s):
         if not s:
@@ -2737,28 +2753,28 @@ def _normalize_double_desc_amount_header_table(grid):
 
     out = [["Date", "Description", "Amount"]]
 
-    for row in grid[1:]:
+    for row in grid[hdr_idx + 1 :]:
         cells = [str(c or "").strip() for c in row]
-        while len(cells) < 5:
+        while len(cells) < ncols:
             cells.append("")
-        c0, c1, c2, c3, c4 = cells[:5]
+        c0, c1, c2 = cells[0], cells[1] if len(cells) > 1 else "", cells[2] if len(cells) > 2 else ""
+        c3 = cells[3] if len(cells) > 3 else ""
+        c4 = cells[4] if len(cells) > 4 else ""
 
-        # Skip completely empty rows
-        if not any(cells[:5]):
+        if not any(cells[:ncols]):
             continue
 
         # A) Fused date + label in col0, amount in col1 (Beginning/Ending Balance, etc.)
         mpre = date_prefix_re.match(c0)
         if mpre and _is_money(c1):
             date_s = mpre.group(1).strip()
-            tail = (mpre.group(2) or "").strip()
-            low = (c0 + " " + tail).lower()
+            low = (c0 + " " + (mpre.group(2) or "")).lower()
             if "beginning balance" in low:
                 desc = "Beginning Balance"
             elif "ending balance" in low:
                 desc = "Ending Balance"
             else:
-                desc = tail if tail else "Transaction"
+                desc = (mpre.group(2) or "").strip() or "Transaction"
             out.append([date_s, desc, c1])
             continue
 
@@ -2782,12 +2798,71 @@ def _normalize_double_desc_amount_header_table(grid):
             out.append([date_s, desc, amt])
             continue
 
-        # C) Default: use first Date/Description/Amount triple; ignore empty second pair
+        # D) Summary / label rows: first column is NOT a date, second is money (Credits/Debits/Service)
+        if not date_prefix_re.match(c0) and not date_only_re.match(c0.strip()) and _is_money(c1):
+            out.append(["", c0, c1])
+            continue
+
+        # C) Default: first Date|Description|Amount triple (ignore duplicate pair)
         out.append([c0, c1, c2])
 
     return out if len(out) >= 2 else grid
 
 
+def _normalize_ucb_three_col_beginning_balance_fusion(grid):
+    """
+    Fix 14: UCB sometimes OCRs the summary as 3 columns (not 5), with the real
+    beginning balance inside the description and $0.00 in the amount column:
+      02/01/2025 | Beginning Balance $3,562.49 Average Ledger Balance | $0.00
+    Normalize to: Date | Beginning Balance | $3,562.49
+    Guard: standard Date|Description|Amount header only.
+    """
+    if not grid or len(grid) < 2:
+        return grid
+
+    def _tight(s):
+        return re.sub(r"\s+", "", str(s or "").strip().lower())
+
+    h0 = [str(c or "").strip() for c in grid[0]]
+    # Do not use max row width — OCR sometimes adds an extra empty column on some rows only.
+    if len(h0) < 3:
+        return grid
+    if _tight(h0[0]) != "date" or _tight(h0[1]) != "description" or _tight(h0[2]) != "amount":
+        return grid
+
+    date_only_re = re.compile(r"^\d{1,2}/\d{1,2}/\d{2,4}$")
+
+    def _extract_money(s):
+        if not s:
+            return ""
+        m = re.search(r"(-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?)", str(s))
+        return m.group(1).strip() if m else ""
+
+    zeroish = {"", "$0.00", "0.00", "$0", ".00", "$.00", "0"}
+
+    out = [list(grid[0])]
+    changed = False
+    for row in grid[1:]:
+        cells = [str(c or "").strip() for c in row]
+        while len(cells) < 3:
+            cells.append("")
+        c0, c1, c2 = cells[0], cells[1], cells[2]
+        if (
+            date_only_re.match(c0.strip())
+            and c1
+            and "beginning balance" in c1.lower()
+        ):
+            mx = _extract_money(c1)
+            c2n = c2.strip() if c2 else ""
+            if mx and (not c2n or c2n in zeroish):
+                out.append([c0.strip(), "Beginning Balance", mx])
+                changed = True
+                continue
+        out.append([c0, c1, c2])
+
+    return out if changed else grid
+
+
 def _normalize_fused_date_posted_amount_table(grid):
     """
     Normalize 2-column OCR tables with fused header like:
@@ -2915,12 +2990,15 @@ def normalize_html_tables(text: str) -> str:
                 last = m.end()  # must set here before continue since line below is outside try
                 continue
 
+            # Fix 13: MUST run before _drop_truly_empty_columns — otherwise the 5th column
+            # can be removed and the UCB double-pair header no longer matches.
+            grid = _normalize_double_desc_amount_header_table(grid)
+            # Fix 14: same bank, 3-col OCR fused Beginning Balance + $0.00 amount column.
+            grid = _normalize_ucb_three_col_beginning_balance_fusion(grid)
+
             grid = _drop_truly_empty_columns(grid)
             grid = _merge_blank_header_text_columns(grid)
 
-            # Fix 13: Date|Description|Amount|Description|Amount → 3 columns (UCB-style)
-            grid = _normalize_double_desc_amount_header_table(grid)
-
             # Fix 5: reconstruct mid-table separator rows split across columns by OCR
             grid = _reconstruct_separator_rows(grid)
 
@@ -3192,6 +3270,10 @@ def run_ocr(uploaded_file):
                 # Safe no-op for scanned PDFs (no text layer) and non-transaction pages.
                 if is_pdf and not is_nfcu_doc:
                     stabilized = _patch_ocr_with_textlayer(stabilized, path, page_num)
+                    # _patch_ocr_with_textlayer may append Case B rows from the PDF text layer
+                    # with fused UCB-style cells; run the same HTML table pass again (Fix 13/14).
+                    stabilized = normalize_html_tables(stabilized)
+                    stabilized = _dedupe_subset_transaction_tables_html(stabilized)
                 elif is_pdf and is_nfcu_doc:
                     # Navy-only additive fix: rebuild malformed summary table from
                     # text layer, but keep GLM body/header/footer unchanged.