Update app.py

app.py

@@ -119,6 +119,12 @@ Primary goals for reconcile rate:
             five columns using trailing amount in the description cell and TD-specific debit/credit
             routing. Fires on \"continued\" in-table text and/or TD body cues (TD ZELLE, LENDINGCLUB,
             etc.). Safe no-op elsewhere.
+- Fix SE-CONT: CO-OP / credit-union account banner (BUSINESS - N [CONT]) often emits three columns
+            | Amount | Amount | (date+desc | amount | balance) or a four-column variant (date+desc
+            fragment | desc tail | amount | balance). CONT may be omitted by OCR. Downstream mis-maps
+            amount vs balance. Normalize to Date | Transaction Description | Transaction Amount | Balance.
+- Fix DA4-dedupe: In 4-column Date|…|Amount|Balance registers, consecutive rows identical on all four
+            cells (same running balance) are OCR stutters — collapse to one; honour text-layer counts when present.
 - Fix MD-1: Markdown fidelity — (a) collapse adjacent HTML tables that are the same register
             (identical multiset of date + debit + credit + balance per row, descriptions may differ);
             keep the table with richer description text so output matches the PDF once. (b) Common OCR
@@ -2972,6 +2978,187 @@ def _fix_three_col_date_desc_amount_left_shift(grid):
     return out if changed else grid
 
 
+# CO-OP account banner: digits + optional CONT (OCR often truncates "CONT")
+_BUSINESS_COOP_BANNER_FIRST_CELL_RE = re.compile(
+    r"^BUSINESS\s*-\s*\d+(?:\s*CONT)?\s*$",
+    re.IGNORECASE,
+)
+
+
+def _first_cell_is_business_coop_banner(s: str) -> bool:
+    t = str(s or "").strip()
+    return bool(t and _BUSINESS_COOP_BANNER_FIRST_CELL_RE.match(t))
+
+
+def _fix_business_coop_cont_three_col_register_grid(grid):
+    """
+    Fix SE-CONT: CO-OP network continuation (BUSINESS-…CONT | Amount | Amount) where col1 is
+    MM/DD + description, col2 is transaction amount, col3 is running balance. Downstream
+    extraction treats the two money columns like Amount/Balance in a different column order,
+    inflating debits/credits. Expand to the same 4-column register as the first page.
+    """
+    if not grid or len(grid) < 3:
+        return grid
+
+    def _tight_cell(s: str) -> str:
+        return re.sub(r"\s+", "", str(s or "").strip().lower())
+
+    hdr_idx = None
+    scan = min(5, len(grid))
+    for ri in range(scan):
+        cells = [str(c or "").strip() for c in grid[ri]]
+        while len(cells) < 3:
+            cells.append("")
+        if len(cells) < 3:
+            continue
+        if _first_cell_is_business_coop_banner(cells[0]) and _tight_cell(
+            cells[1]
+        ) == "amount" and _tight_cell(cells[2]) == "amount":
+            hdr_idx = ri
+            break
+    if hdr_idx is None:
+        return grid
+
+    date_desc_re = re.compile(r"^(\d{1,2}/\d{1,2})\s+(.+)$", re.DOTALL)
+    money_re = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
+
+    def _is_date_txn_desc_subheader(cells: List[str]) -> bool:
+        c0 = str(cells[0] or "").strip().lower()
+        return (
+            "date" in c0
+            and "transaction" in c0
+            and "description" in c0
+            and len(c0) < 80
+        )
+
+    data_start = hdr_idx + 1
+    if data_start < len(grid):
+        r1 = [str(c or "").strip() for c in grid[data_start]]
+        while len(r1) < 3:
+            r1.append("")
+        if _is_date_txn_desc_subheader(r1):
+            data_start += 1
+
+    new_grid = [
+        ["Date", "Transaction Description", "Transaction Amount", "Balance"],
+    ]
+    changed = False
+    for row in grid[data_start:]:
+        cells = [str(c or "").strip() for c in row]
+        while len(cells) < 3:
+            cells.append("")
+        c0, c1, c2 = cells[0], cells[1], cells[2]
+        if not c0:
+            continue
+        m = date_desc_re.match(c0.strip())
+        if not m:
+            continue
+        amt = (c1 or "").strip()
+        bal = (c2 or "").strip()
+        if not amt or not bal:
+            continue
+        if not money_re.match(amt) or not money_re.match(bal):
+            continue
+        new_grid.append(
+            [
+                m.group(1).strip(),
+                (m.group(2) or "").strip(),
+                amt,
+                bal,
+            ]
+        )
+        changed = True
+
+    if not changed or len(new_grid) < 2:
+        return grid
+    return new_grid
+
+
+def _fix_business_coop_cont_four_col_register_grid(grid):
+    """
+    Fix SE-CONT (4-col): Same CO-OP continuation banner as 3-col, but OCR uses four columns:
+    BUSINESS-…CONT | (blank) | Amount | Amount, then subheader, then rows where col0 is
+    MM/DD + description start, col1 is description tail (e.g. SEATTLE), col2 amount, col3 balance.
+    Without merging, downstream maps columns incorrectly. Output canonical 4-column register.
+    """
+    if not grid or len(grid) < 3:
+        return grid
+
+    ncols = max(len(r) for r in grid if isinstance(r, list))
+    if ncols < 4:
+        return grid
+
+    def _tight_cell(s: str) -> str:
+        return re.sub(r"\s+", "", str(s or "").strip().lower())
+
+    hdr_idx = None
+    for ri in range(min(6, len(grid))):
+        cells = [str(c or "").strip() for c in grid[ri]]
+        while len(cells) < ncols:
+            cells.append("")
+        if len(cells) < 4:
+            continue
+        if not _first_cell_is_business_coop_banner(cells[0]):
+            continue
+        if _tight_cell(cells[-2]) != "amount" or _tight_cell(cells[-1]) != "amount":
+            continue
+        hdr_idx = ri
+        break
+    if hdr_idx is None:
+        return grid
+
+    date_desc_re = re.compile(r"^(\d{1,2}/\d{1,2})\s+(.+)$", re.DOTALL)
+    money_re = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
+
+    def _skip_subheader(cells: List[str]) -> bool:
+        c0 = str(cells[0] or "").strip()
+        cl = c0.lower()
+        if "date" not in cl or "transaction" not in cl or "description" not in cl:
+            return False
+        return not bool(date_desc_re.match(c0.strip()))
+
+    data_start = hdr_idx + 1
+    while data_start < len(grid):
+        cells = [str(c or "").strip() for c in grid[data_start]]
+        while len(cells) < ncols:
+            cells.append("")
+        if _skip_subheader(cells):
+            data_start += 1
+            continue
+        break
+
+    new_grid = [
+        ["Date", "Transaction Description", "Transaction Amount", "Balance"],
+    ]
+    changed = False
+    for row in grid[data_start:]:
+        cells = [str(c or "").strip() for c in row]
+        while len(cells) < ncols:
+            cells.append("")
+        c0 = cells[0] if cells else ""
+        c1 = cells[1] if len(cells) > 1 else ""
+        c2 = cells[2] if len(cells) > 2 else ""
+        c3 = cells[3] if len(cells) > 3 else ""
+        if not c0:
+            continue
+        m = date_desc_re.match(c0.strip())
+        if not m:
+            continue
+        amt = (c2 or "").strip()
+        bal = (c3 or "").strip()
+        if not amt or not bal:
+            continue
+        if not money_re.match(amt) or not money_re.match(bal):
+            continue
+        desc = ((m.group(2) or "").strip() + " " + (c1 or "").strip()).strip()
+        new_grid.append([m.group(1).strip(), desc, amt, bal])
+        changed = True
+
+    if not changed or len(new_grid) < 2:
+        return grid
+    return new_grid
+
+
 def _normalize_fused_date_posted_amount_table(grid):
     if not grid or len(grid) < 2:
         return grid
@@ -3441,6 +3628,8 @@ def normalize_html_tables(text: str) -> str:
             grid = _normalize_double_desc_amount_header_table(grid)
             grid = _normalize_ucb_three_col_beginning_balance_fusion(grid)
             grid = _fix_three_col_date_desc_amount_left_shift(grid)
+            grid = _fix_business_coop_cont_three_col_register_grid(grid)
+            grid = _fix_business_coop_cont_four_col_register_grid(grid)
 
             grid = _drop_truly_empty_columns(grid)
             grid = _merge_blank_header_text_columns(grid)
@@ -3482,6 +3671,7 @@ def normalize_html_tables(text: str) -> str:
 
             grid = _fix_fused_keyvalue_rows(grid)
             grid = _dedupe_duplicate_rows_in_da3_table(grid)
+            grid = _dedupe_consecutive_identical_da4_register_rows(grid)
             split_tables = _split_combined_summary_daily_balance_grid(grid)
             if split_tables:
                 out.append("\n\n".join(_grid_to_html(g) for g in split_tables if g))
@@ -3657,6 +3847,22 @@ def _is_da3_header(grid):
     )
 
 
+def _is_da4_date_desc_amount_balance_header(grid):
+    """Date | …Description… | …Amount… | …Balance… (4-column running-balance register)."""
+    if not grid or len(grid[0]) < 4:
+        return False
+    h = [re.sub(r"\s+", " ", str(c or "").strip().lower()) for c in grid[0][:4]]
+    if h[0] != "date":
+        return False
+    if "description" not in h[1]:
+        return False
+    if "amount" not in h[2]:
+        return False
+    if "balance" not in h[3]:
+        return False
+    return True
+
+
 _DA3_STRICT_AMOUNT = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*\.\d{2}-?$")
 _DA3_MONEY_TOKEN = re.compile(
     r"(?<![\d,])(-?\$?\d{1,3}(?:,\d{3})*\.\d{2}|-?\$?\d{2,6}\.\d{2})(?!\d)"
@@ -3774,6 +3980,67 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
     return out
 
 
+def _da4_register_row_key(cells: List[str]) -> Tuple[str, str, str, str]:
+    parts = [str(c or "").strip() for c in cells[:4]]
+    while len(parts) < 4:
+        parts.append("")
+    d0, d1, d2, d3 = parts[0], parts[1], parts[2], parts[3]
+    date_s = re.sub(r"\s+", "", d0)
+    amt_s = re.sub(r"[\s$,]", "", d2).lower().replace("−", "-")
+    bal_s = re.sub(r"[\s$,]", "", d3).lower().replace("−", "-")
+    desc_s = html.unescape(d1)
+    desc_s = desc_s.replace("`", "'").replace("'", "'")
+    desc_s = re.sub(r"\s+", "", desc_s.lower())
+    return (date_s, desc_s, amt_s, bal_s)
+
+
+def _dedupe_consecutive_identical_da4_register_rows(grid):
+    """
+    Fix SE-CONT / generic: OCR often repeats the same register line twice at page breaks
+    (identical date, description, amount, and running balance). Real back-to-back
+    duplicates would show different balances. Collapse consecutive full-row matches;
+    honour text-layer counts when >1 (same 3-col key as Fix 20).
+    """
+    if not grid or len(grid) < 2:
+        return grid
+    if not _is_da4_date_desc_amount_balance_header(grid):
+        return grid
+
+    data_rows = []
+    for row in grid[1:]:
+        cells = [str(c or "").strip() for c in row]
+        if not any(cells):
+            continue
+        data_rows.append(row)
+
+    if not data_rows:
+        return grid
+
+    out = [grid[0]]
+    i = 0
+    while i < len(data_rows):
+        row = data_rows[i]
+        cells = [str(c or "").strip() for c in row]
+        k4 = _da4_register_row_key(cells)
+        j = i + 1
+        while j < len(data_rows):
+            ncells = [str(c or "").strip() for c in data_rows[j]]
+            if _da4_register_row_key(ncells) != k4:
+                break
+            j += 1
+        run_len = j - i
+        k3 = _transaction_row_dedupe_key(cells)
+        tl_confirmed = _tl_freq_context.get(k3, 0)
+        # Default cap 1 for identical running-balance rows; allow more if PDF text layer says so.
+        cap = max(tl_confirmed, 1) if tl_confirmed > 0 else 1
+        take = min(run_len, cap)
+        for t in range(take):
+            out.append(data_rows[i + t])
+        i = j
+
+    return out
+
+
 def _row_text_full_ucb(grid, r: int) -> str:
     if not grid or r < 0 or r >= len(grid):
         return ""