Update app.py

app.py

@@ -1,9 +1,7 @@
 """
 GLM-OCR Hugging Face Space app for PDF/image OCR with header inclusion
 and table-structure stabilization for downstream bank-statement pipelines.
-
 Hard-coded knobs (no environment variables required).
-
 Primary goals for reconcile rate:
 - preserve right-most columns (often "Balance") by higher DPI render + right padding
 - keep tables as tables (convert markdown pipe tables -> HTML table)
@@ -11,8 +9,9 @@ Primary goals for reconcile rate:
 - normalize HTML tables generically so downstream parsing/classification is stable:
   1) expand colspan/rowspan into a rectangular grid
   2) drop truly-empty columns (common in summary tables)
-  3) merge “blank header” columns that contain text into the left column (common when DESCRIPTION is split)
+  3) merge "blank header" columns that contain text into the left column (common when DESCRIPTION is split)
   4) clean common header artifacts (e.g. "DESCRIPTIONBeginning Balance", "BALANCE$3,447.10")
+  5) recover fused first data row generically (two-signal guard: text fusion + money fusion must both fire)
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -52,7 +51,6 @@ FORMATTER_PATH = os.path.join(GLMOCR_BASE, "postprocess", "result_formatter.py")
 # HARD-CODED SETTINGS (edit these numbers to tune quality/speed)
 # ============================================================
 
-# IMPORTANT: Do NOT hard-code secrets in a public Space.
 GLMOCR_API_KEY = "e2b138b2005a41cb9d87dd18805838aa.lyd51L23rcDbsw0w"
 
 # Higher = better OCR for small/right-aligned digits; slower
@@ -423,7 +421,6 @@ def _drop_truly_empty_columns(grid):
             total += 1
             if str(row[i] or "").strip():
                 non_empty += 1
-        # keep column if it has meaningful content; otherwise drop
         if total > 0 and (non_empty / total) <= 0.05:
             keep[i] = False
 
@@ -480,7 +477,6 @@ def _merge_blank_header_text_columns(grid):
         non_empty_ratio = non_empty / total
         texty_ratio = (texty / non_empty) if non_empty else 0.0
 
-        # Merge only when it behaves like description continuation text
         if non_empty_ratio >= 0.55 and texty_ratio >= 0.70:
             for r in range(1, len(grid)):
                 row = grid[r]
@@ -501,41 +497,164 @@ def _merge_blank_header_text_columns(grid):
     return new_grid
 
 
-def _clean_header_artifacts(grid):
+# --------------------------
+# Generic fused-header recovery
+# --------------------------
+
+# Matches a standalone money value: optional currency symbol, optional sign,
+# digits with optional comma grouping, optional decimal cents.
+# Covers: $3,447.10  -1,234.56  $-500.00  1000  etc.
+_MONEY_RE = re.compile(
+    r"^[\$£€¥]?\s*-?\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?$"
+)
+
+# All column-header keywords we recognise across any bank statement format.
+# Each entry is a raw regex fragment (no anchors) used to detect the keyword
+# at the START of a header cell.
+_HEADER_KW_PATTERNS = [
+    r"DATE",
+    r"POSTING\s+DATE",
+    r"VALUE\s+DATE",
+    r"TRANSACTION(?:\s+(?:ID|DATE|TYPE|NO|NUMBER))?",
+    r"DESCRIPTION",
+    r"DETAILS?",
+    r"PARTICULARS?",
+    r"NARRATION",
+    r"REMARKS?",
+    r"REF(?:ERENCE)?(?:\s*(?:NO|NUM|NUMBER))?",
+    r"CHEQUE(?:\s*(?:NO|NUMBER))?",
+    r"AMOUNT",
+    r"DEBIT",
+    r"CREDIT",
+    r"DEPOSIT",
+    r"WITHDRAWAL",
+    r"BALANCE",
+    r"RUNNING\s+BALANCE",
+    r"AVAILABLE\s+BALANCE",
+    r"LEDGER\s+BALANCE",
+]
+
+# Pre-compile: each pattern anchored at start, case-insensitive
+_HEADER_KW_RES = [
+    re.compile(r"^(" + p + r")(.*)", re.IGNORECASE | re.DOTALL)
+    for p in _HEADER_KW_PATTERNS
+]
+
+# A set version for quick "is this entire string a known keyword?" check
+_HEADER_KW_EXACT_RE = re.compile(
+    r"^(?:" + r"|".join(_HEADER_KW_PATTERNS) + r")$",
+    re.IGNORECASE,
+)
+
+
+def _split_keyword_remainder(cell_text: str):
     """
-    Generic header cleanup:
-    - DESCRIPTIONBeginning Balance / DESCRIPTIONEnding Balance -> DESCRIPTION
-    - BALANCE$3,447.10 -> BALANCE
+    If cell_text starts with a known header keyword followed by extra content,
+    return (keyword, remainder).  Otherwise return (cell_text, "").
     """
-    if not grid or not grid[0]:
-        return grid
-    hdr = [str(c or "").strip() for c in grid[0]]
+    s = cell_text.strip()
+    for pattern in _HEADER_KW_RES:
+        m = pattern.match(s)
+        if m:
+            keyword = m.group(1).strip()
+            remainder = m.group(2).strip()
+            return keyword, remainder
+    return s, ""
+
+
+def _extract_fused_header_artifacts(header_row):
+    """
+    Generic detector for the OCR artifact where the first data row of a table
+    gets fused into the header cells during OCR.
+
+    The artifact pattern:
+      - At least one header cell contains a known column keyword PLUS free text
+        that is NOT itself a keyword  →  "text-fused" column
+        e.g.  "DESCRIPTIONBeginning Balance"
+              "NARRATIONOpening Balance"
+              "DETAILSForward Balance"
+      - At least one OTHER header cell contains a known column keyword PLUS a
+        money amount  →  "money-fused" column
+        e.g.  "BALANCE$3,447.10"
+              "AMOUNT 5,000.00"
+
+    Both signals must fire simultaneously (two-signal guard).  If only one
+    fires the header is left completely unchanged and None is returned —
+    this prevents false positives on clean PDFs from any bank.
+
+    Returns:
+        (cleaned_header : list[str], recovered_row : list[str] | None)
+    """
+    if not header_row:
+        return list(header_row), None
+
+    ncols = len(header_row)
+    cleaned = list(header_row)
+    recovered = [""] * ncols
 
-    for idx, cell in enumerate(hdr):
-        c = cell
+    text_fused = []   # column indices where a descriptive label was fused
+    money_fused = []  # column indices where a money amount was fused
 
-        if re.search(r"\bDESCRIPTION\b", c, flags=re.IGNORECASE) and re.search(
-            r"\b(Beginning|Ending)\s+Balance\b", c, flags=re.IGNORECASE
-        ):
-            hdr[idx] = "DESCRIPTION"
+    for idx, cell in enumerate(header_row):
+        cell_s = str(cell or "").strip()
+        if not cell_s:
             continue
 
-        if re.search(r"\bBALANCE\b", c, flags=re.IGNORECASE) and re.search(
-            r"\$?\s*-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", c
-        ):
-            hdr[idx] = "BALANCE"
+        keyword, remainder = _split_keyword_remainder(cell_s)
 
-    grid[0] = hdr
-    return grid
+        if not remainder:
+            # Nothing fused into this cell — leave it alone.
+            continue
+
+        # Determine what kind of remainder this is.
+        remainder_no_space = remainder.replace(" ", "")
+
+        if _MONEY_RE.match(remainder_no_space):
+            # Money amount fused into this header cell.
+            cleaned[idx] = keyword
+            recovered[idx] = remainder
+            money_fused.append(idx)
+
+        elif not _HEADER_KW_EXACT_RE.match(remainder.split()[0] if remainder.split() else ""):
+            # Free descriptive text fused into this header cell.
+            # (We exclude the case where the remainder is itself a keyword,
+            #  which would just be a two-word column name like "POSTING DATE"
+            #  partially matched — those are handled by the longer patterns.)
+            cleaned[idx] = keyword
+            recovered[idx] = remainder
+            text_fused.append(idx)
+
+    # Two-signal guard: only inject when BOTH kinds of fusion are present.
+    if text_fused and money_fused:
+        return cleaned, recovered
+
+    # One or zero signals — do not touch anything.
+    return list(header_row), None
+
+
+def _clean_header_artifacts(grid):
+    """
+    Entry point called from normalize_html_tables.
+    Applies _extract_fused_header_artifacts to the header row of the grid.
+
+    Returns (grid, recovered_row | None).
+    The caller inserts recovered_row at position [1] when it is not None.
+    """
+    if not grid or not grid[0]:
+        return grid, None
+
+    cleaned_header, recovered_row = _extract_fused_header_artifacts(grid[0])
+    grid[0] = cleaned_header
+    return grid, recovered_row
 
 
 def normalize_html_tables(text: str) -> str:
     """
     For every <table>...</table>:
       - parse to grid (expand colspan/rowspan)
-      - drop truly-empty columns (summary tables extra blanks)
-      - merge blank-header text columns into the left column (DESCRIPTION split)
-      - clean common header artifacts
+      - drop truly-empty columns
+      - merge blank-header text columns into the left column
+      - clean header artifacts and recover fused first data row (generic)
       - emit normalized <table> HTML
     """
     if not text or "<table" not in text.lower():
@@ -554,7 +673,16 @@ def normalize_html_tables(text: str) -> str:
             grid = _build_grid(p.rows)
             grid = _drop_truly_empty_columns(grid)
             grid = _merge_blank_header_text_columns(grid)
-            grid = _clean_header_artifacts(grid)
+
+            # Generic fused-header artifact recovery.
+            # recovered_row is only non-None when the two-signal guard passes
+            # (both a text-fused cell and a money-fused cell were detected).
+            # On all clean PDFs this returns None → no side effects.
+            grid, recovered_row = _clean_header_artifacts(grid)
+
+            if recovered_row is not None:
+                grid.insert(1, recovered_row)
+
             out.append(_grid_to_html(grid) if grid else table_html)
         except Exception:
             out.append(table_html)