Update app.py

app.py

@@ -8,24 +8,22 @@ Primary goals for reconcile rate:
 - preserve right-most columns (often "Balance") by higher DPI render + right padding
 - keep tables as tables (convert markdown pipe tables -> HTML table)
 - return ---page-separator--- between pages
-- normalize HTML tables:
-  - expand colspan/rowspan to a rectangular grid
-  - merge “blank header” columns into DESCRIPTION when they contain text (common in many statements)
-  - clean common header artifacts (e.g. "DESCRIPTIONBeginning Balance", "BALANCE$3,447.10")
-  - DO NOT hardcode for any single bank/PDF
+- normalize HTML tables generically so downstream parsing/classification is stable:
+  1) expand colspan/rowspan into a rectangular grid
+  2) drop truly-empty columns (common in summary tables)
+  3) merge “blank header” columns that contain text into the left column (common when DESCRIPTION is split)
+  4) clean common header artifacts (e.g. "DESCRIPTIONBeginning Balance", "BALANCE$3,447.10")
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
 import asyncio
 try:
     _orig_close = asyncio.BaseEventLoop.close
-
     def _safe_close(self):
         try:
             _orig_close(self)
         except (ValueError, OSError):
             pass
-
     asyncio.BaseEventLoop.close = _safe_close
 except Exception:
     pass
@@ -86,11 +84,7 @@ def get_parser():
     global _parser
     if _parser is None:
         from glmocr import GlmOcr
-
-        _parser = GlmOcr(
-            api_key=GLMOCR_API_KEY,
-            mode="maas",
-        )
+        _parser = GlmOcr(api_key=GLMOCR_API_KEY, mode="maas")
     return _parser
 
 
@@ -112,14 +106,10 @@ try:
     with open(FORMATTER_PATH, "r") as f:
         source = f.read()
     for label in (
-        '"header"',
-        "'header'",
-        '"footer"',
-        "'footer'",
-        '"doc_header"',
-        "'doc_header'",
-        '"doc_footer"',
-        "'doc_footer'",
+        '"header"', "'header'",
+        '"footer"', "'footer'",
+        '"doc_header"', "'doc_header'",
+        '"doc_footer"', "'doc_footer'",
     ):
         source = re.sub(r",\s*" + re.escape(label), "", source)
         source = re.sub(re.escape(label) + r"\s*,", "", source)
@@ -150,7 +140,6 @@ def get_header_footer_zones(regions, norm_height=1000):
 def extract_zone_text_pdf(pdf_path, page_num, y_start_frac, y_end_frac):
     try:
         import pymupdf as fitz
-
         doc = fitz.open(pdf_path)
         page = doc[page_num]
         h, w = page.rect.height, page.rect.width
@@ -165,7 +154,6 @@ def extract_zone_text_pdf(pdf_path, page_num, y_start_frac, y_end_frac):
 def extract_pdf_text_in_band(pdf_path, page_num, y_start_frac, y_end_frac):
     try:
         import pymupdf as fitz
-
         doc = fitz.open(pdf_path)
         page = doc[page_num]
         h = page.rect.height
@@ -188,7 +176,6 @@ def ocr_zone(image_path, y_start_frac, y_end_frac):
     zone_name = "header" if y_end_frac < 0.5 else "footer"
     try:
         from PIL import Image
-
         img = Image.open(image_path).convert("RGB")
         w, h = img.size
         y0 = max(0, int(h * y_start_frac))
@@ -242,7 +229,7 @@ def fix_account_number(hdr: str) -> str:
     if acct_match:
         acct = acct_match.group(1)
         if hdr.startswith(acct):
-            hdr = hdr[len(acct) :].lstrip()
+            hdr = hdr[len(acct):].lstrip()
     return hdr
 
 
@@ -406,50 +393,55 @@ def _grid_to_html(grid):
 def _is_amount_like(s: str) -> bool:
     if not s:
         return False
-    s2 = s.strip()
-    return re.fullmatch(r"\$?-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", s2) is not None
+    return re.fullmatch(r"\$?-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", s.strip()) is not None
 
 
-def _clean_header_artifacts(grid):
+def _drop_truly_empty_columns(grid):
     """
-    Generic header text cleanup:
-    - DESCRIPTIONBeginning Balance -> DESCRIPTION
-    - DESCRIPTIONEnding Balance -> DESCRIPTION
-    - BALANCE$3,447.10 -> BALANCE
+    Drop columns that are empty in header AND almost always empty in body.
+    This fixes summary tables that have an extra blank trailing column.
     """
-    if not grid or not grid[0]:
+    if not grid or len(grid) < 1:
         return grid
-    hdr = [str(c or "") for c in grid[0]]
 
-    for idx, cell in enumerate(hdr):
-        c = cell.strip()
+    header = [str(c or "").strip() for c in grid[0]]
+    ncols = len(header)
+    if ncols <= 1:
+        return grid
 
-        if re.search(r"\bDESCRIPTION\b", c, flags=re.IGNORECASE) and re.search(
-            r"\b(Beginning|Ending)\s+Balance\b", c, flags=re.IGNORECASE
-        ):
-            hdr[idx] = "DESCRIPTION"
+    body = grid[1:]
+    keep = [True] * ncols
+
+    for i in range(ncols):
+        if header[i] != "":
             continue
+        total = 0
+        non_empty = 0
+        for row in body:
+            if i >= len(row):
+                continue
+            total += 1
+            if str(row[i] or "").strip():
+                non_empty += 1
+        # keep column if it has meaningful content; otherwise drop
+        if total > 0 and (non_empty / total) <= 0.05:
+            keep[i] = False
 
-        if re.search(r"\bBALANCE\b", c, flags=re.IGNORECASE) and re.search(
-            r"\$?\s*-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", c
-        ):
-            # If it contains a number glued to BALANCE, keep label only
-            hdr[idx] = "BALANCE"
+    if all(keep):
+        return grid
 
-    grid[0] = hdr
-    return grid
+    new_grid = []
+    for row in grid:
+        new_grid.append([cell for idx, cell in enumerate(row) if idx < len(keep) and keep[idx]])
+    return new_grid
 
 
-def _collapse_blank_header_columns(grid):
+def _merge_blank_header_text_columns(grid):
     """
-    Generic fix:
-    If the header row has blank columns, and many body rows have non-empty *text* in that blank column,
+    If the header row has blank columns, and most body rows have non-empty *text* in that blank column,
     merge that column into the nearest non-empty header to the left (usually DESCRIPTION),
     then remove the blank column.
-
-    This fixes:
-    - transaction tables where DESCRIPTION is split into two columns (one header blank)
-    - summary tables that have an extra blank trailing column
+    This fixes transaction tables where DESCRIPTION is split across two columns.
     """
     if not grid or len(grid) < 2:
         return grid
@@ -457,7 +449,6 @@ def _collapse_blank_header_columns(grid):
     header = [str(c or "").strip() for c in grid[0]]
     ncols = len(header)
     body = grid[1:]
-
     blank_cols = [i for i, h in enumerate(header) if h == ""]
     if not blank_cols:
         return grid
@@ -465,7 +456,6 @@ def _collapse_blank_header_columns(grid):
     keep = [True] * ncols
 
     for i in blank_cols:
-        # merge target: nearest non-empty header to the left
         j = i - 1
         while j >= 0 and header[j] == "":
             j -= 1
@@ -475,7 +465,6 @@ def _collapse_blank_header_columns(grid):
         total = 0
         non_empty = 0
         texty = 0
-
         for row in body:
             if i >= len(row) or j >= len(row):
                 continue
@@ -488,13 +477,12 @@ def _collapse_blank_header_columns(grid):
 
         if total == 0:
             continue
-
         non_empty_ratio = non_empty / total
-        texty_ratio = (texty / non_empty) if non_empty > 0 else 0.0
+        texty_ratio = (texty / non_empty) if non_empty else 0.0
 
-        # Merge only when this column is mostly text continuation (not numbers)
+        # Merge only when it behaves like description continuation text
         if non_empty_ratio >= 0.55 and texty_ratio >= 0.70:
-            for r in range(1, len(grid)):  # body only
+            for r in range(1, len(grid)):
                 row = grid[r]
                 if i >= len(row) or j >= len(row):
                     continue
@@ -504,22 +492,51 @@ def _collapse_blank_header_columns(grid):
                     row[j] = (left + " " + right).strip() if left else right
             keep[i] = False
 
-    if not all(keep):
-        new_grid = []
-        for row in grid:
-            new_grid.append([cell for idx, cell in enumerate(row) if idx < len(keep) and keep[idx]])
-        return new_grid
+    if all(keep):
+        return grid
+
+    new_grid = []
+    for row in grid:
+        new_grid.append([cell for idx, cell in enumerate(row) if idx < len(keep) and keep[idx]])
+    return new_grid
+
+
+def _clean_header_artifacts(grid):
+    """
+    Generic header cleanup:
+    - DESCRIPTIONBeginning Balance / DESCRIPTIONEnding Balance -> DESCRIPTION
+    - BALANCE$3,447.10 -> BALANCE
+    """
+    if not grid or not grid[0]:
+        return grid
+    hdr = [str(c or "").strip() for c in grid[0]]
 
+    for idx, cell in enumerate(hdr):
+        c = cell
+
+        if re.search(r"\bDESCRIPTION\b", c, flags=re.IGNORECASE) and re.search(
+            r"\b(Beginning|Ending)\s+Balance\b", c, flags=re.IGNORECASE
+        ):
+            hdr[idx] = "DESCRIPTION"
+            continue
+
+        if re.search(r"\bBALANCE\b", c, flags=re.IGNORECASE) and re.search(
+            r"\$?\s*-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", c
+        ):
+            hdr[idx] = "BALANCE"
+
+    grid[0] = hdr
     return grid
 
 
 def normalize_html_tables(text: str) -> str:
     """
     For every <table>...</table>:
-      - parse to grid (expanding colspan/rowspan)
-      - merge blank header columns when they’re actually description continuation
+      - parse to grid (expand colspan/rowspan)
+      - drop truly-empty columns (summary tables extra blanks)
+      - merge blank-header text columns into the left column (DESCRIPTION split)
       - clean common header artifacts
-      - emit normalized <table> HTML without colspan/rowspan
+      - emit normalized <table> HTML
     """
     if not text or "<table" not in text.lower():
         return text
@@ -535,7 +552,8 @@ def normalize_html_tables(text: str) -> str:
             p = TableGridParser()
             p.feed(table_html)
             grid = _build_grid(p.rows)
-            grid = _collapse_blank_header_columns(grid)
+            grid = _drop_truly_empty_columns(grid)
+            grid = _merge_blank_header_text_columns(grid)
             grid = _clean_header_artifacts(grid)
             out.append(_grid_to_html(grid) if grid else table_html)
         except Exception:
@@ -661,7 +679,6 @@ def run_ocr(uploaded_file):
             he = header_end_frac if header_end_frac is not None else DEFAULT_ZONE_FRAC
             fs = footer_start_frac if footer_start_frac is not None else (1.0 - DEFAULT_ZONE_FRAC)
 
-            # clamp
             he = max(0.02, min(0.25, he))
             fs = max(0.75, min(0.98, fs))
 
@@ -701,7 +718,6 @@ def run_ocr(uploaded_file):
 
     except Exception as e:
         import traceback
-
         log.exception("run_ocr failed: %s", e)
         return f"Error: {e}\n\n{traceback.format_exc()}"
 
@@ -716,10 +732,7 @@ def run_ocr(uploaded_file):
 
 with gr.Blocks(title="GLM-OCR") as demo:
     gr.Markdown("# GLM-OCR\nUpload a PDF or image. Headers included; tables stabilized.")
-    file_in = gr.File(
-        label="Upload PDF or image",
-        file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"],
-    )
+    file_in = gr.File(label="Upload PDF or image", file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"])
     run_btn = gr.Button("Run OCR", variant="primary")
     out = gr.Textbox(lines=40, label="Output (markdown)")
     run_btn.click(fn=run_ocr, inputs=file_in, outputs=out)