Update app.py

app.py

@@ -12,6 +12,7 @@ Primary goals for reconcile rate:
   3) merge "blank header" columns that contain text into the left column (common when DESCRIPTION is split)
   4) clean common header artifacts (e.g. "DESCRIPTIONBeginning Balance", "BALANCE$3,447.10")
   5) recover fused first data row generically (two-signal guard: text fusion + money fusion must both fire)
+  6) promote misplaced header row: when real column headers land in a data row, restructure the grid
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -498,54 +499,99 @@ def _merge_blank_header_text_columns(grid):
 
 
 # --------------------------
-# Generic fused-header recovery
+# Shared keyword definitions
 # --------------------------
 
-# Matches a standalone money value: optional currency symbol, optional sign,
-# digits with optional comma grouping, optional decimal cents.
-# Covers: $3,447.10  -1,234.56  $-500.00  1000  etc.
+# Matches a standalone money value across currencies.
+# Covers: $3,447.10  -1,234.56  £500.00  1000  etc.
 _MONEY_RE = re.compile(
     r"^[\$£€¥]?\s*-?\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?$"
 )
 
-# All column-header keywords we recognise across any bank statement format.
-# Each entry is a raw regex fragment (no anchors) used to detect the keyword
-# at the START of a header cell.
+# All column-header keywords recognised across any bank statement format.
 _HEADER_KW_PATTERNS = [
+    # Date variants
     r"DATE",
     r"POSTING\s+DATE",
     r"VALUE\s+DATE",
-    r"TRANSACTION(?:\s+(?:ID|DATE|TYPE|NO|NUMBER))?",
+    r"TXN\s+DATE",
+    r"TRANSACTION\s+DATE",
+    r"ENTRY\s+DATE",
+    r"EFFECTIVE\s+DATE",
+
+    # Transaction ID / reference
+    r"TRANSACTION(?:\s+(?:ID|TYPE|NO|NUMBER))?",
+    r"TXN(?:\s+(?:ID|NO|TYPE))?",
+    r"REF(?:ERENCE)?(?:\s*(?:NO|NUM|NUMBER))?",
+    r"CHEQUE(?:\s*(?:NO|NUMBER))?",
+    r"CHQ(?:\s*(?:NO|NUMBER))?",
+    r"VOUCHER(?:\s*(?:NO|NUMBER))?",
+    r"SR\.?\s*NO\.?",
+    r"SERIAL(?:\s*(?:NO|NUMBER))?",
+
+    # Description variants
     r"DESCRIPTION",
     r"DETAILS?",
-    r"PARTICULARS?",
+    r"PARTICULARS?(?:\s+OF\s+TRANSACTION)?",
     r"NARRATION",
     r"REMARKS?",
-    r"REF(?:ERENCE)?(?:\s*(?:NO|NUM|NUMBER))?",
-    r"CHEQUE(?:\s*(?:NO|NUMBER))?",
-    r"AMOUNT",
+    r"NOTES?",
+
+    # Debit variants
     r"DEBIT",
+    r"DEBITS?",
+    r"DR\.?",
+    r"WITHDRAWALS?",
+    r"PAID\s+OUT",
+    r"MONEY\s+OUT",
+
+    # Credit variants
     r"CREDIT",
-    r"DEPOSIT",
-    r"WITHDRAWAL",
+    r"CREDITS?",
+    r"CR\.?",
+    r"DEPOSITS?",
+    r"PAID\s+IN",
+    r"MONEY\s+IN",
+
+    # Amount
+    r"AMOUNT",
+
+    # Balance variants
     r"BALANCE",
+    r"BAL\.?",
     r"RUNNING\s+BALANCE",
+    r"RUNNING\s+BAL\.?",
     r"AVAILABLE\s+BALANCE",
+    r"AVAILABLE\s+BAL\.?",
+    r"AVAIL\.?\s+BAL\.?",
     r"LEDGER\s+BALANCE",
+    r"LEDGER\s+BAL\.?",
+    r"CLOSING\s+BALANCE",
+    r"CLOSING\s+BAL\.?",
+    r"OPENING\s+BALANCE",
+    r"OPENING\s+BAL\.?",
 ]
 
-# Pre-compile: each pattern anchored at start, case-insensitive
+# Pre-compiled: each pattern anchored at start, case-insensitive
 _HEADER_KW_RES = [
     re.compile(r"^(" + p + r")(.*)", re.IGNORECASE | re.DOTALL)
     for p in _HEADER_KW_PATTERNS
 ]
 
-# A set version for quick "is this entire string a known keyword?" check
+# Quick exact-match check: "is this entire string a known keyword?"
 _HEADER_KW_EXACT_RE = re.compile(
     r"^(?:" + r"|".join(_HEADER_KW_PATTERNS) + r")$",
     re.IGNORECASE,
 )
 
+# Minimum fraction of cells in a row that must be pure keywords for that row
+# to be considered a misplaced header row.
+_HEADER_ROW_KEYWORD_THRESHOLD = 0.5
+
+
+# --------------------------
+# Fix 1: Fused-header recovery
+# --------------------------
 
 def _split_keyword_remainder(cell_text: str):
     """
@@ -567,20 +613,13 @@ def _extract_fused_header_artifacts(header_row):
     Generic detector for the OCR artifact where the first data row of a table
     gets fused into the header cells during OCR.
 
-    The artifact pattern:
-      - At least one header cell contains a known column keyword PLUS free text
-        that is NOT itself a keyword  →  "text-fused" column
-        e.g.  "DESCRIPTIONBeginning Balance"
-              "NARRATIONOpening Balance"
-              "DETAILSForward Balance"
-      - At least one OTHER header cell contains a known column keyword PLUS a
-        money amount  →  "money-fused" column
-        e.g.  "BALANCE$3,447.10"
-              "AMOUNT 5,000.00"
-
-    Both signals must fire simultaneously (two-signal guard).  If only one
-    fires the header is left completely unchanged and None is returned —
-    this prevents false positives on clean PDFs from any bank.
+    The artifact pattern (BOTH signals must fire simultaneously):
+      - Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
+        e.g.  "DESCRIPTIONBeginning Balance"  "NARRATIONOpening Balance"
+      - Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
+        e.g.  "BALANCE$3,447.10"  "AMOUNT 5,000.00"
+
+    Two-signal guard prevents false positives on clean PDFs from any bank.
 
     Returns:
         (cleaned_header : list[str], recovered_row : list[str] | None)
@@ -592,8 +631,8 @@ def _extract_fused_header_artifacts(header_row):
     cleaned = list(header_row)
     recovered = [""] * ncols
 
-    text_fused = []   # column indices where a descriptive label was fused
-    money_fused = []  # column indices where a money amount was fused
+    text_fused = []
+    money_fused = []
 
     for idx, cell in enumerate(header_row):
         cell_s = str(cell or "").strip()
@@ -603,42 +642,30 @@ def _extract_fused_header_artifacts(header_row):
         keyword, remainder = _split_keyword_remainder(cell_s)
 
         if not remainder:
-            # Nothing fused into this cell — leave it alone.
             continue
 
-        # Determine what kind of remainder this is.
         remainder_no_space = remainder.replace(" ", "")
 
         if _MONEY_RE.match(remainder_no_space):
-            # Money amount fused into this header cell.
             cleaned[idx] = keyword
             recovered[idx] = remainder
             money_fused.append(idx)
 
         elif not _HEADER_KW_EXACT_RE.match(remainder.split()[0] if remainder.split() else ""):
-            # Free descriptive text fused into this header cell.
-            # (We exclude the case where the remainder is itself a keyword,
-            #  which would just be a two-word column name like "POSTING DATE"
-            #  partially matched — those are handled by the longer patterns.)
             cleaned[idx] = keyword
             recovered[idx] = remainder
             text_fused.append(idx)
 
-    # Two-signal guard: only inject when BOTH kinds of fusion are present.
     if text_fused and money_fused:
         return cleaned, recovered
 
-    # One or zero signals — do not touch anything.
     return list(header_row), None
 
 
 def _clean_header_artifacts(grid):
     """
-    Entry point called from normalize_html_tables.
-    Applies _extract_fused_header_artifacts to the header row of the grid.
-
+    Entry point for Fix 1 called from normalize_html_tables.
     Returns (grid, recovered_row | None).
-    The caller inserts recovered_row at position [1] when it is not None.
     """
     if not grid or not grid[0]:
         return grid, None
@@ -648,14 +675,168 @@ def _clean_header_artifacts(grid):
     return grid, recovered_row
 
 
+# --------------------------
+# Fix 2: Misplaced header row promotion
+# --------------------------
+
+def _row_keyword_score(row):
+    """
+    Return the fraction of non-empty cells in this row that are pure header
+    keywords (e.g. DATE, DESCRIPTION, DEBIT, CREDIT, BALANCE).
+    Also handles cells like "DATE DESCRIPTION" where two keywords are space-joined
+    into one cell — these count as a keyword cell too.
+    """
+    non_empty = [str(c or "").strip() for c in row if str(c or "").strip()]
+    if not non_empty:
+        return 0.0
+
+    keyword_hits = 0
+    for cell in non_empty:
+        # Check if the whole cell is a known keyword
+        if _HEADER_KW_EXACT_RE.match(cell):
+            keyword_hits += 1
+            continue
+        # Check if the cell is two or more space-separated keywords
+        # e.g. "DATE DESCRIPTION"
+        parts = cell.split()
+        if all(_HEADER_KW_EXACT_RE.match(p) for p in parts) and len(parts) > 1:
+            keyword_hits += 1
+            continue
+
+    return keyword_hits / len(non_empty)
+
+
+def _split_merged_keyword_cell(cell_text: str, target_ncols: int, col_idx: int, total_cols: int):
+    """
+    When a data cell contains multiple space-separated keywords (e.g. "DATE DESCRIPTION"),
+    split it into individual keyword tokens.  Returns a list of strings to replace
+    the single cell — padded/trimmed to fit the available column space.
+    """
+    parts = cell_text.strip().split()
+    # Only split if every part is a keyword
+    if len(parts) > 1 and all(_HEADER_KW_EXACT_RE.match(p) for p in parts):
+        return parts
+    return [cell_text]
+
+
+def _promote_misplaced_header_row(grid):
+    """
+    Detects and fixes the OCR artifact where the real column headers (DATE,
+    DESCRIPTION, DEBIT, CREDIT, BALANCE …) land in a <td> data row instead
+    of the <th> header row.
+
+    This happens when a bank PDF wraps the transaction section in a larger
+    table that also contains section titles (e.g. "ACCOUNT ACTIVITY" and
+    "Transactions by Date (continued)"), so the OCR emits:
+
+        <th>ACCOUNT ACTIVITY</th>          ← wrong header (section title)
+        <td>Transactions by Date …</td>    ← subtitle row
+        <td>DATE DESCRIPTION</td> <td>DEBIT</td> <td>CREDIT</td> <td>BALANCE</td>
+        <td>01/05 …</td> …                 ← real data
+
+    The fix:
+      1. Scan every data row (rows 1+) for a row whose cells are predominantly
+         pure header keywords.
+      2. When found, promote that row to be the new header (row 0) and discard
+         all rows above it (they are section titles / subtitles).
+      3. If any cell in the promoted row contains multiple space-joined keywords
+         (e.g. "DATE DESCRIPTION"), expand it into separate columns and
+         re-align the data rows below accordingly.
+
+    Guard conditions (no-op if not met — safe for all other PDFs):
+      - The current header row must NOT already look like real column headers
+        (keyword score < threshold).
+      - At least one data row must have keyword score >= threshold.
+      - The candidate row must appear within the first 5 data rows (it is
+        always near the top of the table, never buried mid-table).
+    """
+    if not grid or len(grid) < 2:
+        return grid
+
+    current_header = grid[0]
+    current_header_score = _row_keyword_score(current_header)
+
+    # If the current header already looks like real column headers, do nothing.
+    if current_header_score >= _HEADER_ROW_KEYWORD_THRESHOLD:
+        return grid
+
+    # Search the first few data rows for a candidate header row.
+    candidate_idx = None
+    candidate_score = 0.0
+    search_limit = min(len(grid), 6)  # rows 1..5
+
+    for i in range(1, search_limit):
+        score = _row_keyword_score(grid[i])
+        if score >= _HEADER_ROW_KEYWORD_THRESHOLD and score > candidate_score:
+            candidate_score = score
+            candidate_idx = i
+
+    if candidate_idx is None:
+        return grid
+
+    # --- Promote the candidate row ---
+    candidate_row = grid[candidate_idx]
+
+    # Expand any merged keyword cells (e.g. "DATE DESCRIPTION" → ["DATE", "DESCRIPTION"])
+    expanded_header = []
+    for cell in candidate_row:
+        cell_s = str(cell or "").strip()
+        parts = cell_s.split()
+        if len(parts) > 1 and all(_HEADER_KW_EXACT_RE.match(p) for p in parts):
+            expanded_header.extend(parts)
+        else:
+            expanded_header.append(cell_s)
+
+    new_ncols = len(expanded_header)
+    old_ncols = len(candidate_row)
+    col_expansion = new_ncols - old_ncols  # how many extra columns were added
+
+    # Re-align data rows below the candidate (expand first cell if needed)
+    new_data_rows = []
+    for row in grid[candidate_idx + 1:]:
+        if col_expansion > 0 and row:
+            # The first cell of a data row often contains DATE + start of DESCRIPTION
+            # fused together (e.g. "01/05 TD ZELLE RECEIVED,...").
+            # Split it into [date_part, description_part] to fill the expanded columns.
+            first_cell = str(row[0] or "").strip()
+            date_match = re.match(r"^(\d{1,2}/\d{1,2})\s+(.*)", first_cell, re.DOTALL)
+            if date_match and col_expansion == 1:
+                new_row = [date_match.group(1), date_match.group(2).strip()] + list(row[1:])
+            else:
+                # Can't split cleanly — pad with empty cells to maintain column count
+                new_row = list(row) + [""] * col_expansion
+            new_data_rows.append(new_row)
+        else:
+            new_data_rows.append(list(row))
+
+    # Normalise all rows to new_ncols width
+    def pad(row, n):
+        r = list(row)
+        while len(r) < n:
+            r.append("")
+        return r[:n]
+
+    new_grid = [pad(expanded_header, new_ncols)]
+    for row in new_data_rows:
+        new_grid.append(pad(row, new_ncols))
+
+    return new_grid
+
+
+# --------------------------
+# Normalizer entry point
+# --------------------------
+
 def normalize_html_tables(text: str) -> str:
     """
     For every <table>...</table>:
-      - parse to grid (expand colspan/rowspan)
-      - drop truly-empty columns
-      - merge blank-header text columns into the left column
-      - clean header artifacts and recover fused first data row (generic)
-      - emit normalized <table> HTML
+      1. Parse to grid (expand colspan/rowspan).
+      2. Drop truly-empty columns.
+      3. Merge blank-header text columns into the left column.
+      4. Fix 1 — clean fused header artifacts and recover the lost first data row.
+      5. Fix 2 — promote a misplaced header row when real column headers landed
+                  in a <td> data row instead of the <th> header row.
+      6. Emit normalised <table> HTML.
     """
     if not text or "<table" not in text.lower():
         return text
@@ -674,15 +855,14 @@ def normalize_html_tables(text: str) -> str:
             grid = _drop_truly_empty_columns(grid)
             grid = _merge_blank_header_text_columns(grid)
 
-            # Generic fused-header artifact recovery.
-            # recovered_row is only non-None when the two-signal guard passes
-            # (both a text-fused cell and a money-fused cell were detected).
-            # On all clean PDFs this returns None → no side effects.
+            # Fix 1: fused-header artifact (two-signal guard — safe on clean PDFs)
             grid, recovered_row = _clean_header_artifacts(grid)
-
             if recovered_row is not None:
                 grid.insert(1, recovered_row)
 
+            # Fix 2: misplaced header row promotion (keyword-score guard — safe on clean PDFs)
+            grid = _promote_misplaced_header_row(grid)
+
             out.append(_grid_to_html(grid) if grid else table_html)
         except Exception:
             out.append(table_html)