fragments
- (e.g. 2293 $6,721.00 4/29/2025). Collapse detected gallery pages into one Check #|Amount|Date
- table when bbox/image markers + repeated check lines are present; skips chunks with YOUR CHECKS SEQUENCED.
-- Fix MD-4: YOUR CHECKS SEQUENCED wide grid (N×DATE|CHECK#|AMOUNT per row) is unfolded to one DATE|CHECK#|AMOUNT
- table so narrow viewers show every check, not only the first date column.
-- Fix FH-1: Drop a leading Date|Description|Amount (DA3) table when its (date, amount) multiset is a
- strict subset of a later DEPOSIT/WITHDRAWAL register (e.g. First Horizon OCR duplicate before
- ACCOUNT SUMMARY). Safe no-op when no such pair exists.
-- Fix FH-2: Normalize packed DAILY BALANCE SUMMARY and CHECKS PAID SUMMARY tables into one-row-per-item
- layouts; also promote multiline register headers where row0 is placeholder and row1 has true
- DATE/DESCRIPTION/DEPOSIT/WITHDRAWAL labels.
-- Fix SE-multi: Multi-account deposit summary tables (Beginning/Deposits/Withdrawals/Ending) sometimes get a
- bogus OCR sub-row with tiny amounts (e.g. -$1 / $2 / $1) next to a real account row. Drop those
- dust rows when another row has statement-scale balances, then recompute the TOTAL row from the
- remaining account rows. Generic; no institution names.
+Simplified GLM-OCR Hugging Face / local Gradio app.
+
+Scope (intentionally small):
+ - PDF → padded high-DPI page images → GLM-OCR body markdown
+ - Header band: PDF text extraction first, optional header OCR fallback
+ - Footer band: same pattern, with light dedup so we do not paste a full
+ transaction dump twice when the body already captured it
+
+Explicitly omitted vs the heavy Space build:
+ - No HTML table rewriting, text-layer row injection, institution-specific
+ splits (UCB / Navy / TD / First Horizon / …), or doc-wide dedupe passes
+
+Configure GLMOCR_API_KEY (environment variable). Optional: glmocr + gradio +
+pymupdf + pillow installed.
"""
-# Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
+# Patch asyncio first (before Gradio imports it) to reduce Python 3.13 loop noise
import asyncio
+
try:
_orig_close = asyncio.BaseEventLoop.close
+
def _safe_close(self):
try:
_orig_close(self)
except (ValueError, OSError):
pass
+
asyncio.BaseEventLoop.close = _safe_close
except Exception:
pass
+import html
import logging
import os
import re
-import html
import tempfile
from typing import List, Optional, Tuple
-from collections import Counter, defaultdict
-from html.parser import HTMLParser
import yaml
-# Gradio and glmocr are optional at import: pro_processor only needs stabilize_tables_and_text
-# (glm_ocr_client). HF Space / run_ocr still use glmocr + gradio when installed.
try:
import glmocr
GLMOCR_BASE = os.path.dirname(glmocr.__file__)
CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
- FORMATTER_PATH = os.path.join(GLMOCR_BASE, "postprocess", "result_formatter.py")
except ImportError:
glmocr = None # type: ignore
GLMOCR_BASE = ""
CONFIG_PATH = ""
- FORMATTER_PATH = ""
-log = logging.getLogger("glmocr_app")
+log = logging.getLogger("glmocr_simple_app")
logging.basicConfig(level=logging.INFO)
-# ============================================================
-# HARD-CODED SETTINGS (edit these numbers to tune quality/speed)
-# ============================================================
-
-GLMOCR_API_KEY = "cee1d52dd91a4ab591b3f6e105f8ad89.LgbQTECuzX0zrito"
+# ---------------------------------------------------------------------------
+# Settings
+# ---------------------------------------------------------------------------
-# Higher = better OCR for small/right-aligned digits; slower
-RENDER_SCALE = 2.2 # try 2.5 if right-side numbers are missed
+GLMOCR_API_KEY = os.environ.get("GLMOCR_API_KEY", "").strip()
+if not GLMOCR_API_KEY:
+ log.warning("GLMOCR_API_KEY is not set; GlmOcr() will fail until you export it.")
-# Padding to protect columns near edges (Balance is often right-most)
+RENDER_SCALE = 2.2
PAD_LEFT_FRAC = 0.02
-PAD_RIGHT_FRAC = 0.06 # try 0.10 if right-most balances are missing
+PAD_RIGHT_FRAC = 0.06
PAD_TOP_FRAC = 0.01
PAD_BOTTOM_FRAC = 0.01
-
ENABLE_CONTRAST = True
DEFAULT_ZONE_FRAC = 0.12
PDF_HEADER_BAND_FRAC = 0.10
-ENABLE_FOOTER_OCR = True # enabled — 3-step fallback with dedup guard
+ENABLE_FOOTER_OCR = True
PDF_FOOTER_BAND_FRAC = 0.88
MIN_CROP_HEIGHT = 112
MIN_CROP_PIXELS = 112 * 112
-# ============================================================
-
_parser = None
+
def get_parser():
global _parser
+ if glmocr is None:
+ raise RuntimeError("glmocr is not installed.")
if _parser is None:
from glmocr import GlmOcr
+
_parser = GlmOcr(api_key=GLMOCR_API_KEY, mode="maas")
return _parser
-# ---------------------------------------------------------------------------
-# Best-effort config tweaks (safe to fail on read-only HF env)
-# ---------------------------------------------------------------------------
+
if CONFIG_PATH:
try:
- with open(CONFIG_PATH, "r") as f:
+ with open(CONFIG_PATH, "r", encoding="utf-8") as f:
config = yaml.safe_load(f)
+ config.setdefault("pipeline", {}).setdefault("maas", {})
config["pipeline"]["maas"]["enabled"] = True
config["pipeline"]["maas"]["api_key"] = GLMOCR_API_KEY
- with open(CONFIG_PATH, "w") as f:
+ with open(CONFIG_PATH, "w", encoding="utf-8") as f:
yaml.dump(config, f, default_flow_style=False, sort_keys=False)
except Exception:
pass
-# Best-effort formatter tweak: avoid stripping header/footer labels
-if FORMATTER_PATH:
- try:
- with open(FORMATTER_PATH, "r") as f:
- source = f.read()
- for label in (
- '"header"', "'header'",
- '"footer"', "'footer'",
- '"doc_header"', "'doc_header'",
- '"doc_footer"', "'doc_footer'",
- ):
- source = re.sub(r",\s*" + re.escape(label), "", source)
- source = re.sub(re.escape(label) + r"\s*,", "", source)
- source = re.sub(re.escape(label), "", source)
- with open(FORMATTER_PATH, "w") as f:
- f.write(source)
- except Exception:
- pass
-# --------------------------
-# Header/footer helpers
-# --------------------------
def get_header_footer_zones(regions, norm_height=1000):
if not regions:
return None, None
@@ -281,9 +118,11 @@ def get_header_footer_zones(regions, norm_height=1000):
return None, None
return min(y_tops) / norm_height, max(y_bottoms) / norm_height
+
def extract_zone_text_pdf(pdf_path, page_num, y_start_frac, y_end_frac):
try:
import pymupdf as fitz
+
doc = fitz.open(pdf_path)
page = doc[page_num]
h, w = page.rect.height, page.rect.width
@@ -294,9 +133,11 @@ def extract_zone_text_pdf(pdf_path, page_num, y_start_frac, y_end_frac):
except Exception:
return ""
+
def extract_pdf_text_in_band(pdf_path, page_num, y_start_frac, y_end_frac):
try:
import pymupdf as fitz
+
doc = fitz.open(pdf_path)
page = doc[page_num]
h = page.rect.height
@@ -314,10 +155,12 @@ def extract_pdf_text_in_band(pdf_path, page_num, y_start_frac, y_end_frac):
except Exception:
return ""
+
def ocr_zone(image_path, y_start_frac, y_end_frac):
zone_name = "header" if y_end_frac < 0.5 else "footer"
try:
from PIL import Image
+
img = Image.open(image_path).convert("RGB")
w, h = img.size
y0 = max(0, int(h * y_start_frac))
@@ -359,6 +202,7 @@ def ocr_zone(image_path, y_start_frac, y_end_frac):
log.warning("[%s] ocr_zone failed: %s", zone_name, e, exc_info=True)
return ""
+
def fix_account_number(hdr: str) -> str:
if not hdr:
return hdr
@@ -370,12 +214,10 @@ def fix_account_number(hdr: str) -> str:
if acct_match:
acct = acct_match.group(1)
if hdr.startswith(acct):
- hdr = hdr[len(acct):].lstrip()
+ hdr = hdr[len(acct) :].lstrip()
return hdr
-# --------------------------
-# Table stabilization helpers
-# --------------------------
+
def close_unclosed_html(md: str) -> str:
if not md:
return md
@@ -392,6 +234,7 @@ def close_unclosed_html(md: str) -> str:
md += ("" % tag) * (opened - closed)
return md
+
def looks_like_markdown_table(block: str) -> bool:
lines = [ln.rstrip() for ln in block.strip().splitlines() if ln.strip()]
if len(lines) < 2:
@@ -401,6 +244,7 @@ def looks_like_markdown_table(block: str) -> bool:
sep = lines[1].replace(" ", "")
return ("---" in sep) and ("|" in sep)
+
def md_table_to_html(block: str) -> str:
lines = [ln.strip() for ln in block.strip().splitlines() if ln.strip()]
if len(lines) < 2:
@@ -423,9 +267,12 @@ def md_table_to_html(block: str) -> str:
cols = split_row(ln)
if len(cols) < len(header):
cols += [""] * (len(header) - len(cols))
- html_rows.append("
" + "".join(f"| {html.escape(c)} | " for c in cols[: len(header)]) + "
")
+ html_rows.append(
+ "
" + "".join(f"| {html.escape(c)} | " for c in cols[: len(header)]) + "
"
+ )
return "
\n" + "\n".join(html_rows) + "\n
"
+
def normalize_money_glyphs(text: str) -> str:
if not text:
return text
@@ -443,5945 +290,191 @@ def normalize_money_glyphs(text: str) -> str:
t = re.sub(r"\b[0-9Oo\$,.\-]{4,}\b", o_to_zero, t)
return t
-# ---- Generic HTML table normalizer ----
-class TableGridParser(HTMLParser):
- """Parse a
into rows of (text, colspan, rowspan)."""
-
- def __init__(self):
- super().__init__()
- self.rows = []
- self._current_row = []
- self._cell_text = []
- self._colspan = 1
- self._rowspan = 1
- self._in_cell = False
-
- def handle_starttag(self, tag, attrs):
- if tag == "tr":
- self._current_row = []
- elif tag in ("td", "th"):
- attrs_d = dict(attrs)
- self._colspan = max(1, int(attrs_d.get("colspan", 1)))
- self._rowspan = max(1, int(attrs_d.get("rowspan", 1)))
- self._cell_text = []
- self._in_cell = True
-
- def handle_endtag(self, tag):
- if tag in ("td", "th"):
- text = "".join(self._cell_text).strip().replace("\n", " ")
- self._current_row.append((text, self._colspan, self._rowspan))
- self._in_cell = False
- elif tag == "tr":
- self.rows.append(self._current_row)
-
- def handle_data(self, data):
- if self._in_cell:
- self._cell_text.append(data)
-
-def _build_grid(rows_data):
- if not rows_data:
- return []
- blocked = defaultdict(set)
- grid = []
-
- for r, row_cells in enumerate(rows_data):
- grid.append([])
- col = 0
- for content, C, R in row_cells:
- while col in blocked[r]:
- grid[r].append("")
- col += 1
- for k in range(C):
- grid[r].append(content if k == 0 else "")
- for k in range(1, R):
- blocked[r + k].add(col)
- col += C
-
- max_cols = max(len(row) for row in grid) if grid else 0
- for row in grid:
- while len(row) < max_cols:
- row.append("")
- return grid
-
-def _grid_to_html(grid):
- if not grid:
- return ""
- lines = [""]
- for r, row in enumerate(grid):
- lines.append("")
- tag = "th" if r == 0 else "td"
- for cell in row:
- escaped = (
- (cell or "")
- .replace("&", "&")
- .replace("<", "<")
- .replace(">", ">")
- .replace('"', """)
- )
- lines.append(f"<{tag}>{escaped}")
- lines.append("
")
- lines.append("
")
- return "\n".join(lines)
-
-def _is_amount_like(s: str) -> bool:
- if not s:
- return False
- return re.fullmatch(r"\$?-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", s.strip()) is not None
-
-def _drop_truly_empty_columns(grid):
- if not grid or len(grid) < 1:
- return grid
-
- header = [str(c or "").strip() for c in grid[0]]
- ncols = len(header)
- if ncols <= 1:
- return grid
-
- body = grid[1:]
- keep = [True] * ncols
-
- for i in range(ncols):
- if header[i] != "":
- continue
- total = 0
- non_empty = 0
- for row in body:
- if i >= len(row):
- continue
- total += 1
- if str(row[i] or "").strip():
- non_empty += 1
- if total > 0 and (non_empty / total) <= 0.05:
- keep[i] = False
-
- if all(keep):
- return grid
-
- new_grid = []
- for row in grid:
- new_grid.append([cell for idx, cell in enumerate(row) if idx < len(keep) and keep[idx]])
- return new_grid
-
-
-def _is_multigroup_check_list_header(header: List[str]) -> bool:
- """
- Wide check listings (e.g. YOUR CHECKS SEQUENCED) use one row with a title cell, repeated
- Amount headers, and blank slots between DATE | CHECK # | AMOUNT triplets. Fix 3
- blank-header merge would otherwise fuse the next column into Amount or the section title,
- leaving only the first date column visibly populated.
- """
- if not header:
- return False
- joined = " ".join(str(c or "").strip() for c in header).upper()
- if "SEQUENCED" in joined and "CHECK" in joined:
- return True
- # Repeated Amount columns in one header row = multi-group layout; do not merge blanks.
- if sum(1 for c in header if str(c or "").strip().upper() == "AMOUNT") >= 2:
- return True
- return False
-
-
-def _should_skip_blank_header_merge_multiline_register(grid) -> bool:
- """
- Some banks (e.g. First Horizon) use a title row with blank | cells, then a second header
- row with DATE DESCRIPTION | DEPOSIT | WITHDRAWAL | CARD #. _merge_blank_header_text_columns
- would treat those blanks as OCR splits and merge columns — corrupting the grid.
- """
- if not grid or len(grid) < 2:
- return False
- h0_join = " ".join(str(c or "").strip() for c in grid[0]).upper()
- if "ACCOUNT HISTORY" in h0_join:
- return True
- for ri in range(1, min(5, len(grid))):
- cells = [str(c or "").strip().upper() for c in grid[ri]]
- if not any(cells):
- continue
- row_join = " ".join(cells)
- if "DEPOSIT" in row_join and "WITHDRAWAL" in row_join:
- if "DATE" in row_join or "DESCRIPTION" in row_join:
- return True
- return False
-
-
-def _merge_blank_header_text_columns(grid):
- if not grid or len(grid) < 2:
- return grid
-
- header = [str(c or "").strip() for c in grid[0]]
- if _is_multigroup_check_list_header(header):
- return grid
- if _should_skip_blank_header_merge_multiline_register(grid):
- return grid
-
- ncols = len(header)
- body = grid[1:]
- blank_cols = [i for i, h in enumerate(header) if h == ""]
- if not blank_cols:
- return grid
-
- keep = [True] * ncols
-
- for i in blank_cols:
- j = i - 1
- while j >= 0 and header[j] == "":
- j -= 1
- if j < 0:
- continue
-
- total = 0
- non_empty = 0
- texty = 0
- for row in body:
- if i >= len(row) or j >= len(row):
- continue
- v = str(row[i] or "").strip()
- total += 1
- if v:
- non_empty += 1
- if not _is_amount_like(v):
- texty += 1
-
- if total == 0:
- continue
- non_empty_ratio = non_empty / total
- texty_ratio = (texty / non_empty) if non_empty else 0.0
- if non_empty_ratio >= 0.55 and texty_ratio >= 0.70:
- for r in range(1, len(grid)):
- row = grid[r]
- if i >= len(row) or j >= len(row):
- continue
- left = str(row[j] or "").strip()
- right = str(row[i] or "").strip()
- if right:
- row[j] = (left + " " + right).strip() if left else right
- keep[i] = False
-
- if all(keep):
- return grid
-
- new_grid = []
- for row in grid:
- new_grid.append([cell for idx, cell in enumerate(row) if idx < len(keep) and keep[idx]])
- return new_grid
-
-
-def _split_wide_checks_sequenced_grid(grid: List[List[str]]) -> Optional[List[List[str]]]:
- """
- Fix MD-4: Banks print YOUR CHECKS SEQUENCED as 2–3 side-by-side DATE|CHECK #|AMOUNT groups per row.
- OCR HTML keeps that as 6–9 columns; narrow markdown previews only show the first date column.
- Unfold into a single standard 3-column table (one row per check).
- """
- if not grid or len(grid) < 2:
- return None
- header = [str(c or "").strip() for c in grid[0]]
- header_join = " ".join(header).upper()
- if "SEQUENCED" not in header_join or "CHECK" not in header_join:
- return None
- ncols = max(len(r) for r in grid if isinstance(r, list))
- if ncols < 6 or ncols % 3 != 0:
- return None
- amt_cols = sum(1 for c in header if str(c or "").strip().upper() == "AMOUNT")
- if amt_cols < 2 and ncols < 9:
- return None
-
- date_re = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
- out: List[List[str]] = [["DATE", "CHECK #", "AMOUNT"]]
-
- for row in grid[1:]:
- cells = [str(c or "").strip() for c in row]
- while len(cells) < ncols:
- cells.append("")
- cells = cells[:ncols]
- c0 = cells[0] if cells else ""
- joined = " ".join(cells)
-
- if c0.startswith("=") and ("=" * 3 in c0 or len(c0) >= 8):
- continue
- if c0 and "=" in c0 and not c0.replace("=", "").strip():
- continue
- if "DATE" in c0.upper() and "CHECK" in c0.upper() and "AMOUNT" in c0.upper():
- continue
- if "GAP IN CHECK" in joined.upper() and "SEQUENCE" in joined.upper():
- continue
-
- for g in range(0, ncols, 3):
- if g + 2 >= len(cells):
- break
- d, ck, am = cells[g], cells[g + 1], cells[g + 2]
- if not d and not ck and not am:
- continue
- if d and date_re.match(d):
- out.append([d, ck, am])
-
- if len(out) < 2:
- return None
- return out
-
-
-def _fill_headless_trailing_amount_headers(grid):
- """
- Fix 32a: PDF layouts often omit a header above the rightmost amount column; OCR leaves a
- trailing empty | . Downstream parsers then mis-associate columns. Fill only when the
- column is predominantly strict currency (generic, bank-agnostic).
- """
- if not grid or len(grid) < 2:
- return grid
-
- header = [str(c or "").strip() for c in grid[0]]
- ncols = len(header)
- if ncols < 2:
- return grid
-
- i = ncols - 1
- while i >= 0 and header[i] == "":
- i -= 1
- empty_tail_indices = list(range(i + 1, ncols))
- if not empty_tail_indices:
- return grid
-
- body = grid[1:]
- new_header = list(header)
-
- for j in empty_tail_indices:
- total = 0
- amt_hits = 0
- for row in body:
- if j >= len(row):
- continue
- total += 1
- v = str(row[j] or "").strip()
- if v and _is_amount_like(v):
- amt_hits += 1
- if total == 0:
- continue
- if (amt_hits / total) < 0.45:
- continue
-
- left = j - 1
- while left >= 0 and not new_header[left]:
- left -= 1
- prev = (new_header[left] if left >= 0 else "").strip()
- pl = prev.lower()
- if any(
- k in pl
- for k in (
- "addition",
- "subtraction",
- "credit",
- "debit",
- "deposit",
- "withdrawal",
- )
- ):
- new_header[j] = prev
- else:
- new_header[j] = "Amount"
-
- grid[0] = new_header
- return grid
-
-
-_MMDD_DATE_RE = re.compile(r"^\d{1,2}[-/]\d{2}(?:[-/]\d{2,4})?$")
-
-# Leading transaction-type cell when OCR splits category vs merchant (East West / similar).
-_STACKED_TXN_TYPE_RE = re.compile(
- r"^(?:Pre-Auth Credit|Pre-Auth Debit|Preauth Debit|"
- r"Withdrawal Trns|Withdrawal(?:\s|$)|Deposit(?:\s|$)|Deposit Transfe|"
- r"Outgoing Wire|Incoming Wire|"
- r"Service Charge|ACH Debit|ACH Credit|ACH Payment|ACH PMT|"
- r"Wire Transfer|POS Purchase|ATM Withdrawal|Transfer Trns)\b",
- re.IGNORECASE | re.DOTALL,
-)
-
-
-def _normalize_date_desc_money_table_alignment(grid):
- """
- Fix 32b: Align OCR tables with common bank PDF layouts.
-
- * Credits (Additions): Number | Date | Transaction Description | Additions — Number is often
- blank in the PDF but must exist as a column; OCR may omit it or split description across
- columns and park amounts under a bogus \"Amount\" header.
- * Debits (Subtractions): PDF is usually Date | Description | Subtractions (no Number) —
- when OCR uses the same 4-column split error, merge back to three columns.
-
- Handles (a) Date | short type | merchant tail | amount, (b) Date | full desc | empty | amount,
- (c) continuation rows with a blank date. Per-row merge (no single global mode) so mixed OCR
- errors (e.g. wrapped merchant lines) still normalize.
- """
- if not grid or len(grid) < 3:
- return grid
-
- header = [str(c or "").strip() for c in grid[0]]
- n = len(header)
-
- def _pad_row(cells, width):
- cells = [str(c or "").strip() for c in cells]
- while len(cells) < width:
- cells.append("")
- return cells[:width]
-
- h0n = re.sub(r"\s+", " ", header[0]).strip().lower()
- h1n = re.sub(r"\s+", " ", header[1]).strip().lower() if n > 1 else ""
- h2n = re.sub(r"\s+", " ", header[2]).strip().lower() if n > 2 else ""
- h3n = re.sub(r"\s+", " ", header[3]).strip().lower() if n > 3 else ""
-
- money_label = header[2].strip() if n > 2 else ""
- want_number_col = "addition" in h2n
-
- # --- 3-column Date | Transaction Description | Additions/Subtractions: prepend Number for credits only ---
- if n == 3:
- if h0n != "date" or "transaction" not in h1n or "description" not in h1n:
- return grid
- if not any(
- k in h2n
- for k in (
- "addition",
- "subtraction",
- "credit",
- "debit",
- "deposit",
- "withdrawal",
- )
- ):
- return grid
- if not want_number_col:
- return grid
- out = [["Number", header[0], header[1], money_label or "Additions"]]
- for row in grid[1:]:
- c = _pad_row(row, 3)
- if not any(c):
- out.append(["", "", "", ""])
- continue
- out.append(["", c[0], c[1], c[2]])
- return out
-
- if n != 4:
- return grid
-
- if h0n != "date":
- return grid
- if "transaction" not in h1n or "description" not in h1n:
- return grid
- if not any(
- k in h2n
- for k in (
- "addition",
- "subtraction",
- "credit",
- "debit",
- "deposit",
- "withdrawal",
- )
- ):
- return grid
- if h3n not in ("", "amount") and not any(
- k in h3n for k in ("addition", "subtraction", "credit", "debit")
- ):
- return grid
-
- data_rows = [r for r in grid[1:] if any(str(c or "").strip() for c in r)]
- if not data_rows:
- return grid
-
- has_date_led = False
- for row in data_rows:
- c = _pad_row(row, 4)
- if c[0] and _MMDD_DATE_RE.match(c[0]):
- has_date_led = True
- break
- if not has_date_led:
- return grid
-
- def _merge_date_led_row(c0, c1, c2, c3):
- split_ok = (
- bool(c1)
- and bool(_STACKED_TXN_TYPE_RE.match(c1))
- and bool(c2)
- and not _is_amount_like(c2)
- and _is_amount_like(c3)
- )
- if split_ok:
- return (c1 + " " + c2).strip(), c3
- shift_ok = (
- bool(c1)
- and not _is_amount_like(c1)
- and not (c2 or "").strip()
- and _is_amount_like(c3)
- )
- if shift_ok:
- return c1, c3
- dep_ok = (
- bool(c1)
- and bool(_STACKED_TXN_TYPE_RE.match(c1))
- and not (c2 or "").strip()
- and _is_amount_like(c3)
- )
- if dep_ok:
- return c1, c3
- desc = ((c1 + " " + c2).strip() or c1).strip()
- amt = c3 if _is_amount_like(c3) else ""
- return desc, amt
-
- ml = money_label or ("Additions" if want_number_col else "Subtractions")
-
- if want_number_col:
- out_h = ["Number", header[0], header[1], ml]
- else:
- out_h = [header[0], header[1], ml]
-
- out = [out_h]
- for row in grid[1:]:
- c = _pad_row(row, 4)
- c0, c1, c2, c3 = c[0], c[1], c[2], c[3]
- if not any(c):
- out.append([""] * len(out_h))
- continue
- if c0 and _MMDD_DATE_RE.match(c0):
- desc, amt = _merge_date_led_row(c0, c1, c2, c3)
- if want_number_col:
- out.append(["", c0, desc, amt])
- else:
- out.append([c0, desc, amt])
- continue
- if want_number_col:
- desc_t = (c1 + " " + c2).strip()
- amt_t = c3 if _is_amount_like(c3) else ""
- out.append(["", c0, desc_t, amt_t])
+def light_stabilize_markdown(page_md: str) -> str:
+ """Convert obvious GitHub-style pipe tables to HTML; normalize money glyphs; repair tags."""
+ if not page_md:
+ return page_md
+ page_md = normalize_money_glyphs(page_md)
+ blocks = re.split(r"\n\s*\n", page_md.strip())
+ out_blocks = []
+ for b in blocks:
+ if looks_like_markdown_table(b):
+ out_blocks.append(md_table_to_html(b))
else:
- desc_t = (c1 + " " + c2).strip()
- amt_t = c3 if _is_amount_like(c3) else ""
- out.append([c0, desc_t, amt_t])
-
- return out
-
-
-# --------------------------
-# Shared keyword definitions
-# --------------------------
-
-_MONEY_RE = re.compile(
- r"^(?:[\$£€¥]|Rs\.?|INR|PKR)?\s*-?\s?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$",
- re.IGNORECASE,
-)
-
-_HEADER_KW_PATTERNS = [
- r"DATE",
- r"POSTING\s+DATE",
- r"VALUE\s+DATE",
- r"TXN\s+DATE",
- r"TRANSACTION\s+DATE",
- r"ENTRY\s+DATE",
- r"EFFECTIVE\s+DATE",
- r"TRANSACTION(?:\s+(?:ID|TYPE|NO|NUMBER))?",
- r"TXN(?:\s+(?:ID|NO|TYPE))?",
- r"REF(?:ERENCE)?(?:\s*(?:NO|NUM|NUMBER))?",
- r"CHEQUE(?:\s*(?:NO|NUMBER))?",
- r"CHQ(?:\s*(?:NO|NUMBER))?",
- r"VOUCHER(?:\s*(?:NO|NUMBER))?",
- r"SR\.?\s*NO\.?",
- r"SERIAL(?:\s*(?:NO|NUMBER))?",
- r"DESCRIPTION",
- r"DETAILS?",
- r"PARTICULARS?(?:\s+OF\s+TRANSACTION)?",
- r"NARRATION",
- r"REMARKS?",
- r"NOTES?",
- r"DEBIT",
- r"DEBITS?",
- r"DR\.?",
- r"WITHDRAWALS?",
- r"PAID\s+OUT",
- r"MONEY\s+OUT",
- r"CREDIT",
- r"CREDITS?",
- r"CR\.?",
- r"DEPOSITS?",
- r"PAID\s+IN",
- r"MONEY\s+IN",
- r"AMOUNT",
- r"BALANCE",
- r"BAL\.?",
- r"RUNNING\s+BALANCE",
- r"RUNNING\s+BAL\.?",
- r"AVAILABLE\s+BALANCE",
- r"AVAILABLE\s+BAL\.?",
- r"AVAIL\.?\s+BAL\.?",
- r"LEDGER\s+BALANCE",
- r"LEDGER\s+BAL\.?",
- r"CLOSING\s+BALANCE",
- r"CLOSING\s+BAL\.?",
- r"OPENING\s+BALANCE",
- r"OPENING\s+BAL\.?",
-]
-
-_HEADER_KW_RES = [
- re.compile(r"^(" + p + r")(.*)", re.IGNORECASE | re.DOTALL)
- for p in _HEADER_KW_PATTERNS
-]
-
-_HEADER_KW_EXACT_RE = re.compile(
- r"^(?:" + r"|".join(_HEADER_KW_PATTERNS) + r")$",
- re.IGNORECASE,
-)
-
-_HEADER_ROW_KEYWORD_THRESHOLD = 0.5
-
-# --------------------------
-# Fix 1: Fused-header recovery
-# --------------------------
-
-def _split_keyword_remainder(cell_text: str):
- s = cell_text.strip()
- for pattern in _HEADER_KW_RES:
- m = pattern.match(s)
- if m:
- keyword = m.group(1).strip()
- remainder = m.group(2).strip()
- return keyword, remainder
- return s, ""
-
-def _extract_fused_header_artifacts(header_row):
- if not header_row:
- return list(header_row), None
-
- ncols = len(header_row)
- cleaned = list(header_row)
- recovered = [""] * ncols
-
- text_fused = []
- money_fused = []
-
- for idx, cell in enumerate(header_row):
- cell_s = str(cell or "").strip()
- if not cell_s:
- continue
-
- keyword, remainder = _split_keyword_remainder(cell_s)
-
- if not remainder:
- continue
-
- remainder_no_space = remainder.replace(" ", "")
-
- if _MONEY_RE.match(remainder_no_space):
- cleaned[idx] = keyword
- recovered[idx] = remainder
- money_fused.append(idx)
-
- elif not _HEADER_KW_EXACT_RE.match(remainder.split()[0] if remainder.split() else ""):
- cleaned[idx] = keyword
- recovered[idx] = remainder
- text_fused.append(idx)
-
- if text_fused and money_fused:
- return cleaned, recovered
-
- return list(header_row), None
+ out_blocks.append(b)
+ return close_unclosed_html("\n\n".join(out_blocks))
-def _clean_header_artifacts(grid):
- if not grid or not grid[0]:
- return grid, None
- cleaned_header, recovered_row = _extract_fused_header_artifacts(grid[0])
- grid[0] = cleaned_header
- return grid, recovered_row
+def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
+ import pymupdf as fitz
+ from PIL import Image, ImageEnhance
-# --------------------------
-# Fix 2: Misplaced header row promotion
-# --------------------------
+ doc = fitz.open(pdf_path)
+ page_images: List[str] = []
+ page_heights: List[int] = []
-def _row_keyword_score(row):
- non_empty = [str(c or "").strip() for c in row if str(c or "").strip()]
- if not non_empty:
- return 0.0
+ for i in range(len(doc)):
+ page = doc[i]
+ pix = page.get_pixmap(matrix=fitz.Matrix(RENDER_SCALE, RENDER_SCALE), alpha=False)
- keyword_hits = 0
- for cell in non_empty:
- if _HEADER_KW_EXACT_RE.match(cell):
- keyword_hits += 1
- continue
- parts = cell.split()
- if all(_HEADER_KW_EXACT_RE.match(p) for p in parts) and len(parts) > 1:
- keyword_hits += 1
- continue
+ img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
- return keyword_hits / len(non_empty)
+ if ENABLE_CONTRAST:
+ img = ImageEnhance.Contrast(img).enhance(1.12)
-_BALANCE_INFO_RE = re.compile(
- r"(beginning|ending|opening|closing)\s+balance"
- r"|account\s*#?\s*\d{5,}"
- r"|\b\d{7,}\b"
- r"|\$\d{1,3}(?:,\d{3})*\.\d{2}",
- re.IGNORECASE,
-)
+ w, h = img.size
+ pad_l = int(w * PAD_LEFT_FRAC)
+ pad_r = int(w * PAD_RIGHT_FRAC)
+ pad_t = int(h * PAD_TOP_FRAC)
+ pad_b = int(h * PAD_BOTTOM_FRAC)
-def _header_contains_balance_info(header_row):
- for cell in header_row:
- if _BALANCE_INFO_RE.search(str(cell or "")):
- return True
- return False
+ if any(p > 0 for p in (pad_l, pad_r, pad_t, pad_b)):
+ canvas = Image.new("RGB", (w + pad_l + pad_r, h + pad_t + pad_b), (255, 255, 255))
+ canvas.paste(img, (pad_l, pad_t))
+ img = canvas
-def _promote_misplaced_header_row(grid):
- if not grid or len(grid) < 2:
- return grid
+ img_path = os.path.join(tempfile.gettempdir(), f"glmocr_page_{os.getpid()}_{i}.png")
+ img.save(img_path, "PNG", compress_level=6)
+ page_images.append(img_path)
+ page_heights.append(img.height)
- current_header_score = _row_keyword_score(grid[0])
- if current_header_score >= _HEADER_ROW_KEYWORD_THRESHOLD:
- return grid
+ doc.close()
+ return page_images, page_heights
- candidate_idx = None
- candidate_score = 0.0
- for i in range(1, min(len(grid), 6)):
- score = _row_keyword_score(grid[i])
- if score >= _HEADER_ROW_KEYWORD_THRESHOLD and score > candidate_score:
- candidate_score = score
- candidate_idx = i
- if candidate_idx is None:
- return grid
+def get_page_md_and_regions(page_result):
+ md = ""
+ if hasattr(page_result, "markdown_result") and page_result.markdown_result:
+ md = (page_result.markdown_result or "").strip()
+ regions = []
+ if hasattr(page_result, "json_result"):
+ jr = page_result.json_result
+ if isinstance(jr, dict) and "regions" in jr:
+ regions = jr.get("regions") or []
+ elif isinstance(jr, list) and len(jr) > 0:
+ r = jr[0] if isinstance(jr[0], list) else jr
+ if isinstance(r, list):
+ regions = r
+ elif isinstance(r, dict) and "regions" in r:
+ regions = r.get("regions") or []
+ return md, regions
- candidate_row = grid[candidate_idx]
- expanded_header = []
- for cell in candidate_row:
- cell_s = str(cell or "").strip()
- parts = cell_s.split()
- if len(parts) > 1 and all(_HEADER_KW_EXACT_RE.match(p) for p in parts):
- expanded_header.extend(parts)
- else:
- expanded_header.append(cell_s)
+def run_ocr(uploaded_file):
+ if uploaded_file is None:
+ return "Please upload a file."
- new_ncols = len(expanded_header)
- col_expansion = new_ncols - len(candidate_row)
+ page_images: List[str] = []
+ try:
+ path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
+ is_pdf = path.lower().endswith(".pdf")
+ parser = get_parser()
- metadata_row = None
- if _header_contains_balance_info(grid[0]):
- meta_cells = [str(c or "").strip() for c in grid[0]]
- meta_text = " ".join(c for c in meta_cells if c).strip()
- if meta_text:
- metadata_row = [meta_text] + [""] * (new_ncols - 1)
+ page_heights: List[int] = []
- new_data_rows = []
- for row in grid[candidate_idx + 1:]:
- if col_expansion > 0 and row:
- first_cell = str(row[0] or "").strip()
- date_match = re.match(r"^(\d{1,2}/\d{1,2})\s+(.*)", first_cell, re.DOTALL)
- if date_match and col_expansion == 1:
- new_row = [date_match.group(1), date_match.group(2).strip()] + list(row[1:])
- else:
- new_row = list(row) + [""] * col_expansion
- new_data_rows.append(new_row)
+ if is_pdf:
+ page_images, page_heights = render_pdf_pages_to_images(path)
+ results = parser.parse(page_images)
else:
- new_data_rows.append(list(row))
+ page_images = [path]
+ page_heights = [1000]
+ results = parser.parse(path)
- def pad(row, n):
- r = list(row)
- while len(r) < n:
- r.append("")
- return r[:n]
+ if not isinstance(results, list):
+ results = [results]
- new_grid = [pad(expanded_header, new_ncols)]
- if metadata_row is not None:
- new_grid.append(pad(metadata_row, new_ncols))
- for row in new_data_rows:
- new_grid.append(pad(row, new_ncols))
+ all_pages = []
+ for page_num, page_result in enumerate(results):
+ page_md, regions = get_page_md_and_regions(page_result)
+ img_h = page_heights[page_num] if page_num < len(page_heights) else 1000
+ header_end_frac, footer_start_frac = get_header_footer_zones(regions, img_h)
- return new_grid
+ he = header_end_frac if header_end_frac is not None else DEFAULT_ZONE_FRAC
+ fs = footer_start_frac if footer_start_frac is not None else (1.0 - DEFAULT_ZONE_FRAC)
-# --------------------------
-# Fix 3: Fused key-value rows in summary sections
-# --------------------------
+ he = max(0.02, min(0.25, he))
+ fs = max(0.75, min(0.98, fs))
-_FUSED_KV_RE = re.compile(
- r"^(.+?)(-?\d{1,3}(?:,\d{3})*(?:\.\d+)?%?)$",
- re.DOTALL,
-)
+ parts = []
-def _fix_fused_keyvalue_rows(grid):
- if not grid or len(grid) < 2:
- return grid
+ hdr = ""
+ if is_pdf:
+ hdr = extract_zone_text_pdf(path, page_num, 0, he)
+ if not (hdr and hdr.strip()):
+ hdr = extract_pdf_text_in_band(path, page_num, 0, PDF_HEADER_BAND_FRAC)
+ if not (hdr and hdr.strip()) and page_num < len(page_images):
+ hdr = ocr_zone(page_images[page_num], 0, he)
+ if hdr and hdr.strip():
+ parts.append(light_stabilize_markdown(fix_account_number(normalize_money_glyphs(hdr.strip()))))
- _DOLLAR_AMOUNT_RE = re.compile(r"\$\s*\d{1,3}(?:,\d{3})*\.\d{2}")
- _LONG_NUMBER_RE = re.compile(r"\d{7,}")
+ if page_md and page_md.strip():
+ parts.append(light_stabilize_markdown(page_md.strip()))
- ncols = len(grid[0])
- new_grid = [grid[0]]
+ if ENABLE_FOOTER_OCR and page_num < len(page_images):
+ ftr = ""
+ if is_pdf:
+ ftr = extract_zone_text_pdf(path, page_num, fs, 1.0)
+ if not (ftr and ftr.strip()):
+ ftr = extract_pdf_text_in_band(path, page_num, PDF_FOOTER_BAND_FRAC, 1.0)
+ if not (ftr and ftr.strip()):
+ ftr = ocr_zone(page_images[page_num], fs, 1.0)
+ if ftr and ftr.strip():
+ ftr_clean = normalize_money_glyphs(ftr.strip())
- for row in grid[1:]:
- cells = [str(c or "").strip() for c in row]
- if not cells:
- new_grid.append(row)
- continue
+ ftr_first_line = next(
+ (ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
+ "",
+ )
+ already_present = ftr_first_line and any(
+ ftr_first_line in part.lower() for part in parts
+ )
- first = cells[0]
- rest_empty = all(c == "" for c in cells[1:])
+ _footer_date_re = re.compile(r"\b\d{1,2}[-/]\d{2}\b")
+ _footer_amt_re = re.compile(r"\b\d{1,3}(?:,\d{3})*\.\d{2}\b")
+ _date_hits = len(_footer_date_re.findall(ftr_clean))
+ _amt_hits = len(_footer_amt_re.findall(ftr_clean))
+ is_txn_dump = _date_hits >= 3 and _amt_hits >= 3
- if not rest_empty or not first:
- new_grid.append(row)
- continue
+ if not already_present and not is_txn_dump:
+ parts.append(ftr_clean)
- if _DOLLAR_AMOUNT_RE.search(first):
- new_grid.append(row)
- continue
+ if parts:
+ all_pages.append("\n\n".join(parts))
- if _LONG_NUMBER_RE.search(first):
- new_grid.append(row)
- continue
+ merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
+ return merged
- if re.search(r"#\s*[\dX]", first):
- new_grid.append(row)
- continue
+ except Exception as e:
+ import traceback
- m = _FUSED_KV_RE.match(first)
- if not m:
- new_grid.append(row)
- continue
+ log.exception("run_ocr failed: %s", e)
+ return f"Error: {e}\n\n{traceback.format_exc()}"
- label_raw = m.group(1)
- value = m.group(2).strip()
+ finally:
+ for p in page_images:
+ try:
+ if isinstance(p, str) and p.endswith(".png") and "glmocr_page_" in os.path.basename(p):
+ os.unlink(p)
+ except Exception:
+ pass
- if label_raw != label_raw.rstrip():
- new_grid.append(row)
- continue
- label = label_raw.strip()
+def _create_gradio_demo():
+ import gradio as gr
- if not label or not value:
- new_grid.append(row)
- continue
-
- new_row = [""] * ncols
- new_row[0] = label
- new_row[ncols - 1] = value
- new_grid.append(new_row)
-
- return new_grid
-
-# --------------------------
-# Fix 4: PDF text-layer row-count patch
-# --------------------------
-
-def _extract_textlayer_rows(pdf_path: str, page_num: int):
- try:
- import pymupdf as fitz
-
- doc = fitz.open(pdf_path)
- page = doc[page_num]
- words = page.get_text("words")
- doc.close()
-
- if not words:
- return []
-
- lines_by_y = {}
- for w in words:
- x0, y0, word = float(w[0]), float(w[1]), w[4]
- bucket = None
- for existing_y in lines_by_y:
- if abs(existing_y - y0) <= 5:
- bucket = existing_y
- break
- if bucket is None:
- bucket = y0
- lines_by_y.setdefault(bucket, []).append((x0, word))
-
- sorted_lines = []
- for y in sorted(lines_by_y):
- line_words = sorted(lines_by_y[y], key=lambda t: t[0])
- sorted_lines.append([w for _, w in line_words])
-
- _MONTHS = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
- date_re = re.compile(
- r"^\d{1,2}/\d{2}(?:/\d{2,4})?$"
- r"|^\d{1,2}-\d{2}(?:-\d{2,4})?$"
- r"|^\d{4}-\d{2}-\d{2}$"
- r"|^" + _MONTHS + r"$",
- re.IGNORECASE,
- )
- month_re = re.compile(r"^" + _MONTHS + r"$", re.IGNORECASE)
- day_re = re.compile(r"^\d{1,2}$")
-
- amount_re = re.compile(
- r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
- r"|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
- r"|^-\s+\$\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
- )
-
- rows = []
- for line in sorted_lines:
- if len(line) < 2:
- continue
-
- date_str = None
- desc_start = None
-
- if date_re.match(line[0]):
- if month_re.match(line[0]) and len(line) > 1 and day_re.match(line[1]):
- date_str = line[0] + " " + line[1]
- desc_start = 2
- else:
- date_str = line[0]
- desc_start = 1
- else:
- continue
-
- if desc_start is None or desc_start >= len(line):
- continue
-
- remaining = line[desc_start:]
- post_date_str = ""
- if remaining and month_re.match(remaining[0]):
- if len(remaining) > 1 and day_re.match(remaining[1]):
- post_date_str = remaining[0] + " " + remaining[1]
- remaining = remaining[2:]
- elif len(remaining) > 0:
- remaining = remaining[1:]
- elif remaining and date_re.match(remaining[0]) and not month_re.match(remaining[0]):
- post_date_str = remaining[0]
- remaining = remaining[1:]
-
- if len(remaining) < 2:
- continue
-
- if (len(remaining) >= 2
- and remaining[-2] == "-"
- and remaining[-1].startswith("$")):
- amount_candidate = remaining[-2] + " " + remaining[-1]
- desc_tokens = remaining[:-2]
- else:
- amount_candidate = remaining[-1]
- desc_tokens = remaining[:-1]
-
- if not amount_re.match(amount_candidate):
- continue
-
- desc = " ".join(desc_tokens).strip()
- if not desc:
- continue
-
- if not re.search(r"[A-Za-z]", desc):
- continue
-
- rows.append({"date": date_str, "post_date": post_date_str, "desc": desc, "amount": amount_candidate})
-
- return rows if len(rows) >= 2 else []
-
- except Exception as e:
- log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
- return []
-
-def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
- if not text_layer:
- return []
- m = re.search(
- r"Checks\s*\([^\n]*\n(.*?)(?=^Daily\s+Account\s+Balance\s*$)",
- text_layer,
- re.DOTALL | re.MULTILINE | re.IGNORECASE,
- )
- if not m:
- return []
- chunk = m.group(1)
- chunk = re.sub(
- r"^\s*Date\s+Number\s+Amount\s*$",
- "",
- chunk,
- flags=re.MULTILINE | re.IGNORECASE,
- )
- _triplet = re.compile(
- r"(\d{2}-\d{2})\s+(\*?\d+)\s+(\d{1,3}(?:,\d{3})*\.\d{2})"
- )
- out: List[Tuple[str, str, str]] = []
- seen = set()
- for t in _triplet.findall(chunk):
- if t not in seen:
- seen.add(t)
- out.append(t)
- return out
-
-def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
- if not text_layer:
- return ""
-
- _DATE_RE = re.compile(r"^\d{2}-\d{2}$")
- _MONEY_RE = re.compile(r"^\d{1,3}(?:,\d{3})*\.\d{2}$")
- _CHECK_RE = re.compile(r"^\*?\d+$")
-
- lines = [l.strip() for l in text_layer.splitlines()]
- output = []
-
- if needs_checks:
- checks = []
- i = 0
- while i < len(lines):
- if lines[i].lower().startswith("checks"):
- i += 1
- while i < len(lines):
- if "daily account balance" in lines[i].lower():
- break
- if lines[i] == "Date":
- i += 1
- dates = []
- while i < len(lines) and _DATE_RE.match(lines[i]):
- dates.append(lines[i]); i += 1
- while i < len(lines) and lines[i] != "Number":
- i += 1
- i += 1
- numbers = []
- while i < len(lines):
- if _CHECK_RE.match(lines[i]):
- numbers.append(lines[i]); i += 1
- elif lines[i] == "":
- i += 1
- else:
- break
- while i < len(lines) and lines[i] != "Amount":
- i += 1
- i += 1
- amounts = []
- while i < len(lines):
- if _MONEY_RE.match(lines[i]):
- amounts.append(lines[i]); i += 1
- elif lines[i] == "":
- i += 1
- else:
- break
- for d, n, a in zip(dates, numbers, amounts):
- checks.append((d, n, a))
- else:
- i += 1
- break
- i += 1
-
- if not checks and re.search(r"\bChecks\s*\(", text_layer, re.I):
- checks = _jb_extract_checks_fallback(text_layer)
-
- if checks:
- rows = "\n".join(
- " | | " + d + " | " + n + " | " + a + " |
"
- for d, n, a in checks
- )
- output.append("Checks")
- output.append(
- '\n| Date | Number | Amount |
|---|
\n'
- + rows + "\n
"
- )
-
- if needs_dab:
- balances = []
- i = 0
- while i < len(lines):
- if "daily account balance" in lines[i].lower():
- i += 1
- while i < len(lines):
- if lines[i] == "Date":
- i += 1
- dates = []
- while i < len(lines) and _DATE_RE.match(lines[i]):
- dates.append(lines[i]); i += 1
- while i < len(lines) and lines[i] != "Balance":
- i += 1
- i += 1
- bals = []
- while i < len(lines):
- if _MONEY_RE.match(lines[i]):
- bals.append(lines[i]); i += 1
- elif lines[i] == "":
- i += 1
- else:
- break
- for d, b in zip(dates, bals):
- balances.append((d, b))
- else:
- i += 1
- break
- i += 1
-
- if balances:
- rows = "\n".join(
- "| " + d + " | " + b + " |
"
- for d, b in balances
- )
- output.append("Daily Account Balance")
- output.append(
- "\n| Date | Balance |
|---|
\n"
- + rows + "\n
"
- )
-
- return "\n\n".join(output)
-
-def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str:
- if not page_md or "]*>.*?
", re.DOTALL | re.IGNORECASE)
- result = page_md
-
- for tbl_match in table_pattern.finditer(page_md):
- table_html = tbl_match.group(0)
-
- p = TableGridParser()
- p.feed(table_html)
- grid = _build_grid(p.rows)
- if len(grid) < 2:
- continue
-
- header = [str(c or "").strip().upper() for c in grid[0]]
- date_col = desc_col = amt_col = None
- for ci, h in enumerate(header):
- if re.search(r"\bDATE\b", h) and date_col is None:
- date_col = ci
- elif re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", h) and desc_col is None:
- desc_col = ci
- elif re.search(r"\b(AMOUNT|DEBIT|CREDIT|DR|CR)\b", h) and amt_col is None:
- amt_col = ci
-
- if date_col is None or desc_col is None or amt_col is None:
- continue
-
- ocr_freq = {}
- ocr_rows_indexed = []
- for ri, row in enumerate(grid[1:], start=1):
- cells = [str(c or "").strip() for c in row]
- if len(cells) <= max(date_col, desc_col, amt_col):
- continue
- d = _norm(cells[date_col])
- desc = _norm(cells[desc_col])
- amt = _norm(cells[amt_col])
- if not d or not desc:
- continue
- key = (d, desc, amt)
- ocr_freq[key] = ocr_freq.get(key, 0) + 1
- ocr_rows_indexed.append((ri, key))
-
- overlap = set(ocr_freq.keys()) & set(tl_freq.keys())
-
- _ocr_dates = [
- _norm(str(row[date_col] or ""))
- for row in grid[1:]
- if len(row) > date_col and str(row[date_col] or "").strip()
- ]
- _tl_dates = [_norm(r["date"]) for r in tl_rows]
- _slash_re = re.compile(r"^\d{1,2}/\d{2}")
- _hyphen_re = re.compile(r"^\d{1,2}-\d{2}")
- def _date_fmt(dates):
- if any(_slash_re.match(d) for d in dates): return "slash"
- if any(_hyphen_re.match(d) for d in dates): return "hyphen"
- return "other"
- ocr_fmt = _date_fmt(_ocr_dates)
- tl_fmt = _date_fmt(_tl_dates)
- date_fmt_match = (ocr_fmt == tl_fmt) or "other" in (ocr_fmt, tl_fmt)
-
- to_inject = {}
- to_inject_new = {}
-
- for key in tl_freq:
- ocr_count = ocr_freq.get(key, 0)
- tl_count = tl_freq[key]
- if tl_count > ocr_count:
- missing = tl_count - ocr_count
- if ocr_count > 0:
- if overlap:
- to_inject[key] = missing
- else:
- if date_fmt_match:
- tl_row_data = next(
- (r for r in tl_rows
- if (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"])) == key),
- None
- )
- if tl_row_data:
- to_inject_new[key] = (missing, tl_row_data)
-
- if not to_inject and not to_inject_new:
- continue
-
- new_grid = [grid[0]]
- for ri, row in enumerate(grid[1:], start=1):
- new_grid.append(row)
- cells = [str(c or "").strip() for c in row]
- if len(cells) <= max(date_col, desc_col, amt_col):
- continue
- d = _norm(cells[date_col])
- desc = _norm(cells[desc_col])
- amt = _norm(cells[amt_col])
- key = (d, desc, amt)
- if key in to_inject and to_inject[key] > 0:
- last_occ = max(idx for idx, k in ocr_rows_indexed if k == key)
- if ri == last_occ:
- for _ in range(to_inject[key]):
- new_grid.append(list(row))
- to_inject[key] = 0
-
- new_table_html = _grid_to_html(new_grid)
- result = result.replace(table_html, new_table_html, 1)
-
- if to_inject_new:
- ncols = len(grid[0])
- header_row = grid[0]
-
- post_date_col = None
- for ci, h in enumerate(header_row):
- hh = str(h or "").strip().upper()
- if "POST" in hh and "DATE" in hh:
- post_date_col = ci
- break
-
- new_rows = []
- for key, (count, tl_row_data) in to_inject_new.items():
- for _ in range(count):
- new_row = [""] * ncols
- new_row[date_col] = tl_row_data["date"]
- new_row[desc_col] = tl_row_data["desc"]
- new_row[amt_col] = tl_row_data["amount"]
- if post_date_col is not None:
- new_row[post_date_col] = tl_row_data.get("post_date", "")
- new_rows.append(new_row)
- log.info(
- "page %d: new table row from text layer: %s %s",
- page_num, tl_row_data["date"], tl_row_data["desc"][:40]
- )
-
- if new_rows:
- extra_grid = [header_row] + new_rows
- extra_html = "\n\n" + _grid_to_html(extra_grid)
- insert_pos = result.find(new_table_html) + len(new_table_html)
- result = result[:insert_pos] + extra_html + result[insert_pos:]
-
- if tl_rows:
- _has_txn_table = False
- for _tbl in table_pattern.finditer(result):
- _p2 = TableGridParser()
- _p2.feed(_tbl.group(0))
- _g2 = _build_grid(_p2.rows)
- if len(_g2) < 2:
- continue
- _h2 = [str(c or "").strip().upper() for c in _g2[0]]
- if (any(re.search(r"\bDATE\b", x) for x in _h2) and
- any(re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", x) for x in _h2)):
- _has_txn_table = True
- break
-
- if not _has_txn_table:
- _has_post = any(r.get("post_date") for r in tl_rows)
- if _has_post:
- _chdr = ["Date", "Post Date", "Transaction Description", "Amount"]
- _di2, _pi2, _xi2, _ai2 = 0, 1, 2, 3
- else:
- _chdr = ["Date", "Transaction Description", "Amount"]
- _di2, _xi2, _ai2 = 0, 1, 2
- _cnew_rows = []
- for r in tl_rows:
- _crow = [""] * len(_chdr)
- _crow[_di2] = r["date"]
- _crow[_xi2] = r["desc"]
- _crow[_ai2] = r["amount"]
- if _has_post:
- _crow[_pi2] = r.get("post_date", "")
- _cnew_rows.append(_crow)
- result = _grid_to_html([_chdr] + _cnew_rows) + "\n\n" + result
- log.info("page %d: Case C — new table (%d rows) from text layer",
- page_num, len(_cnew_rows))
-
- return result
-
- except Exception as e:
- log.warning("_patch_ocr_with_textlayer failed (page %d): %s", page_num, e)
- _tl_freq_context.clear()
- return page_md
-
-# --------------------------
-# Fix 5: Mid-table separator row reconstruction
-# --------------------------
-
-_DATE_RE = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
-_SPLIT_AMOUNT_RE = re.compile(
- r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
-)
-
-def _reconstruct_separator_rows(grid):
- if not grid or len(grid) < 2:
- return grid
-
- ncols = len(grid[0])
- if ncols < 2:
- return grid
-
- new_grid = [grid[0]]
-
- for row in grid[1:]:
- cells = [str(c or "").strip() for c in row]
- while len(cells) < ncols:
- cells.append("")
-
- non_empty = [(i, c) for i, c in enumerate(cells) if c]
-
- if len(non_empty) == 0:
- new_grid.append(row)
- continue
-
- first_idx, first_val = non_empty[0]
-
- if _DATE_RE.match(first_val):
- new_grid.append(row)
- continue
-
- if not re.search(r"[A-Za-z]", first_val):
- new_grid.append(row)
- continue
-
- if len(non_empty) == 1:
- new_row = [""] * ncols
- new_row[0] = first_val
- new_grid.append(new_row)
- continue
-
- trailing = [val for _, val in non_empty[1:]]
- all_trailing_are_digit_fragments = all(
- re.fullmatch(r"\d{1,4}", v) for v in trailing
- )
-
- if not all_trailing_are_digit_fragments:
- new_grid.append(row)
- continue
-
- reconstructed = "".join(val for _, val in non_empty).strip()
- new_row = [""] * ncols
- new_row[0] = reconstructed
- new_grid.append(new_row)
-
- return new_grid
-
-# --------------------------
-# Normalizer entry point
-# --------------------------
-
-def _merge_split_rows(grid):
- if not grid or len(grid) < 3:
- return grid
-
- ncols = len(grid[0])
- if ncols < 3:
- return grid
-
- header = [str(c or "").strip().upper() for c in grid[0]]
- date_col = desc_col = bal_col = None
- debit_cols = []
- credit_cols = []
-
- for ci, h in enumerate(header):
- if re.search(r"\bDATE\b", h) and date_col is None:
- date_col = ci
- if re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?|TRANSACTION)\b", h) and desc_col is None:
- desc_col = ci
- if re.search(r"\bBALANCE\b", h) and bal_col is None:
- bal_col = ci
- if re.search(r"\b(DEBIT|DEBITS|DR|WITHDRAWAL|SUBTRACTIONS?)\b", h):
- debit_cols.append(ci)
- if re.search(r"\b(CREDIT|CREDITS|CR|DEPOSIT|ADDITIONS?)\b", h):
- credit_cols.append(ci)
-
- if date_col is None or desc_col is None:
- return grid
- if not debit_cols and not credit_cols:
- return grid
-
- money_cols = debit_cols + credit_cols
- _MONEY_RE2 = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*\.\d{1,4}$")
- _DATE_RE2 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$|^\d{1,2}-\d{2}(?:-\d{2,4})?$")
- _FRAGMENT_RE = re.compile(r"^[^\s\.]{1,6}$")
-
- new_grid = [grid[0]]
- i = 1
- while i < len(grid):
- row = [str(c or "").strip() for c in grid[i]]
- while len(row) < ncols:
- row.append("")
-
- has_desc = bool(row[desc_col])
- no_date = not row[date_col]
- no_money = all(not row[c] for c in money_cols)
- bal_is_frag = (bal_col is not None and
- (_FRAGMENT_RE.match(row[bal_col]) or not row[bal_col]))
-
- if has_desc and no_date and no_money and bal_is_frag and (i + 1) < len(grid):
- next_row = [str(c or "").strip() for c in grid[i + 1]]
- while len(next_row) < ncols:
- next_row.append("")
-
- next_has_date = bool(next_row[date_col]) and _DATE_RE2.match(next_row[date_col])
- next_has_money = any(bool(next_row[c]) for c in money_cols)
-
- if next_has_date and next_has_money:
- merged = list(next_row)
- if next_row[desc_col]:
- merged[desc_col] = row[desc_col] + " " + next_row[desc_col]
- else:
- merged[desc_col] = row[desc_col]
- new_grid.append(merged)
- i += 2
- continue
-
- new_grid.append(row)
- i += 1
-
- return new_grid
-
-def _extract_fused_desc_amount(grid):
- if not grid or len(grid) < 2:
- return grid
-
- ncols = len(grid[0])
- header = [str(c or "").strip().upper() for c in grid[0]]
-
- date_col = desc_col = None
- debit_cols = []
- credit_cols = []
- for ci, h in enumerate(header):
- if re.search(r"\bDATE\b", h) and date_col is None:
- date_col = ci
- if re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?|TRANSACTION)\b", h) and desc_col is None:
- desc_col = ci
- if re.search(r"\b(DEBIT|DEBITS|DR|WITHDRAWAL|SUBTRACTIONS?)\b", h):
- debit_cols.append(ci)
- if re.search(r"\b(CREDIT|CREDITS|CR|DEPOSIT|ADDITIONS?)\b", h):
- credit_cols.append(ci)
-
- if date_col is None or desc_col is None or not debit_cols:
- return grid
-
- money_cols = debit_cols + credit_cols
- _TRAILING_AMT = re.compile(r"^(.+?)\s+(-?\d{1,3}(?:,\d{3})*\.\d{2})$")
- debit_col = debit_cols[0]
-
- new_grid = [grid[0]]
- for row in grid[1:]:
- cells = [str(c or "").strip() for c in row]
- while len(cells) < ncols:
- cells.append("")
-
- desc = cells[desc_col]
- no_money = all(not cells[c] for c in money_cols)
- has_date = bool(cells[date_col])
-
- if desc and no_money and has_date:
- m = _TRAILING_AMT.match(desc)
- if m:
- cells = list(cells)
- cells[desc_col] = m.group(1).strip()
- cells[debit_col] = m.group(2)
- new_grid.append(cells)
- continue
-
- new_grid.append(cells)
-
- return new_grid
-
-def _is_junk_table(grid):
- if not grid or len(grid) < 2:
- return False
- ncols = len(grid[0])
- if ncols != 1:
- return False
- header_text = str(grid[0][0] or "").strip().upper()
- if "CUSTOMER INFORMATION" in header_text:
- data_rows = grid[1:]
- if all(len(str(r[0] or "").strip()) <= 10 for r in data_rows):
- return True
- return False
-
-def _split_fused_multicolumn_header(grid):
- if not grid or len(grid) < 2:
- return grid
-
- ncols = len(grid[0])
- if ncols < 2:
- return grid
-
- first_header = str(grid[0][0] or "").strip()
- parts = first_header.split()
- kw_hits = [p for p in parts if _HEADER_KW_EXACT_RE.match(p)]
- if len(kw_hits) < 2:
- return grid
-
- rest_headers = [str(c or "").strip() for c in grid[0][1:]]
- if not all(_HEADER_KW_EXACT_RE.match(h) for h in rest_headers if h):
- return grid
-
- new_header_cols = []
- non_kw_parts = []
- for p in parts:
- if _HEADER_KW_EXACT_RE.match(p):
- if non_kw_parts:
- new_header_cols.append(" ".join(non_kw_parts))
- non_kw_parts = []
- new_header_cols.append(p)
- else:
- non_kw_parts.append(p)
- if non_kw_parts:
- new_header_cols.append(" ".join(non_kw_parts))
-
- new_header = new_header_cols + rest_headers
- new_ncols = len(new_header)
- extra_cols = new_ncols - ncols
-
- _DATE_COL_RE = re.compile(r"\bDATE\b", re.IGNORECASE)
- _CARD_COL_RE = re.compile(r"\bCARD\b", re.IGNORECASE)
- _DESC_COL_RE = re.compile(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", re.IGNORECASE)
- _MONEY_COL_RE = re.compile(r"\b(DEPOSIT|WITHDRAWAL|DEBIT|CREDIT|AMOUNT|ADDITION|SUBTRACTION)S?\b", re.IGNORECASE)
-
- new_date_col = next((i for i,h in enumerate(new_header) if _DATE_COL_RE.search(h)), None)
- new_desc_col = next((i for i,h in enumerate(new_header) if _DESC_COL_RE.search(h)), None)
- new_card_col = next((i for i,h in enumerate(new_header) if _CARD_COL_RE.search(h)), None)
-
- if new_date_col is None or new_desc_col is None:
- return grid
-
- _DATE_PREFIX = re.compile(r"^(\d{1,2}/\d{2}(?:/\d{2,4})?)\s+(.*)", re.DOTALL)
- _CARD_SUFFIX = re.compile(r"^(.*?)\s+(\d{4})$", re.DOTALL)
-
- def pad(row, n):
- r = list(row)
- while len(r) < n: r.append("")
- return r[:n]
-
- new_grid = [pad(new_header, new_ncols)]
- for row in grid[1:]:
- cells = [str(c or "").strip() for c in row]
- while len(cells) < ncols: cells.append("")
-
- fused_cell = cells[0]
- rest_cells = cells[1:]
-
- date_val = ""
- desc_val = fused_cell
- card_val = ""
-
- dm = _DATE_PREFIX.match(fused_cell)
- if dm:
- date_val = dm.group(1)
- desc_val = dm.group(2).strip()
-
- if new_card_col is not None:
- cm = _CARD_SUFFIX.match(desc_val)
- if cm:
- desc_val = cm.group(1).strip()
- card_val = cm.group(2)
-
- new_row = [""] * new_ncols
- new_row[new_date_col] = date_val
- new_row[new_desc_col] = desc_val
- if new_card_col is not None:
- new_row[new_card_col] = card_val
- money_col_indices = [i for i,h in enumerate(new_header) if _MONEY_COL_RE.search(h)]
- for mi, ri in enumerate(range(len(rest_cells))):
- if mi < len(money_col_indices):
- new_row[money_col_indices[mi]] = rest_cells[ri]
-
- new_grid.append(new_row)
-
- return new_grid
-
-def _normalize_daily_balance_table(grid):
- if not grid or len(grid) < 2:
- return grid
-
- header = [str(c or "").strip().upper() for c in grid[0]]
- ncols = len(header)
-
- if ncols < 4 or ncols % 2 != 0:
- return grid
-
- half = ncols // 2
- date_half = header[:half]
- bal_half = header[half:]
-
- if not all(h == "DATE" for h in date_half):
- return grid
- if not all(h == "BALANCE" for h in bal_half):
- return grid
-
- new_header = []
- for i in range(half):
- new_header.append("DATE")
- new_header.append("BALANCE")
-
- new_rows = []
- _DATE_RE3 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
-
- for row in grid[1:]:
- cells = [str(c or "").strip() for c in row]
- while len(cells) < ncols:
- cells.append("")
-
- date_cells = [cells[0]]
- bal_cells = cells[1:]
-
- first = date_cells[0]
- tokens = re.split(r"[\n\r\s]+", first)
- date_tokens = [t.strip() for t in tokens if t.strip() and _DATE_RE3.match(t.strip())]
- if len(date_tokens) >= 2:
- stacked = date_tokens
- else:
- stacked = date_cells
-
- new_row = [""] * ncols
- for i, date_val in enumerate(stacked):
- if not _DATE_RE3.match(date_val):
- continue
- bal_val = bal_cells[i] if i < len(bal_cells) else ""
- new_row[i * 2] = date_val
- new_row[i * 2 + 1] = bal_val
- new_rows.append(new_row)
-
- if not new_rows:
- return grid
-
- return [new_header] + new_rows
-
-def _normalize_multiline_register_header(grid):
- """
- Normalize two-row register headers where row0 is mostly blank/Amount placeholders and
- row1 carries the real labels (DATE, DESCRIPTION, DEPOSIT, WITHDRAWAL, CARD #).
- """
- if not grid or len(grid) < 3:
- return grid
- r0 = [str(c or "").strip() for c in grid[0]]
- r1 = [str(c or "").strip() for c in grid[1]]
- if not r1:
- return grid
-
- r1_join = " ".join(r1).upper()
- if "DATE" not in r1_join:
- return grid
- if "WITHDRAWAL" not in r1_join and "DEPOSIT" not in r1_join:
- return grid
- if "DESCRIPTION" not in r1_join:
- return grid
-
- # Guard: only treat as multiline header if row0 is mostly empty / generic title-ish cells.
- non_empty_r0 = [x for x in r0 if x]
- if len(non_empty_r0) > max(3, len(r0) // 2):
- if not ("ACCOUNT HISTORY" in " ".join(r0).upper()):
- return grid
- generic_r0 = all(
- (not x) or (x.upper() in ("AMOUNT", "ACCOUNT HISTORY", "DATE", "DESCRIPTION", "DEPOSIT", "WITHDRAWAL", "CARD #"))
- for x in r0
- )
- if not generic_r0:
- return grid
-
- # Use row1 as canonical header, keeping column count stable.
- ncols = max(len(r0), len(r1))
- new_header = []
- for i in range(ncols):
- v = (r1[i] if i < len(r1) else "").strip()
- if not v and i < len(r0):
- v = str(r0[i] or "").strip()
- new_header.append(v)
-
- out = [new_header]
- out.extend(grid[2:])
- return out
-
-def _normalize_checks_paid_table(grid):
- if not grid or len(grid) < 2:
- return grid
-
- ncols = len(grid[0])
- if ncols < 2:
- return grid
-
- first_h = str(grid[0][0] or "").strip()
- _CHECKS_HDR = re.compile(r"^(DATE\s+CHECK\s+#\s*)+$", re.IGNORECASE)
- if not _CHECKS_HDR.match(first_h):
- return grid
-
- rest_h = [str(c or "").strip().upper() for c in grid[0][1:]]
- if not all(h in ("AMOUNT", "") for h in rest_h if h):
- return grid
-
- n_groups = len(re.findall(r"\bDATE\b", first_h, re.IGNORECASE))
- if n_groups < 1:
- return grid
-
- new_header = []
- for _ in range(n_groups):
- new_header.extend(["DATE", "CHECK #", "AMOUNT"])
-
- _DATE_RE = re.compile(r"^\d{1,2}/\d{2}$")
-
- new_rows = [new_header]
- for row in grid[1:]:
- cells = [str(c or "").strip() for c in row]
- while len(cells) < ncols:
- cells.append("")
-
- fused_data = cells[0]
- amounts = cells[1:]
-
- tokens = fused_data.split()
- groups = []
- current = []
- for tok in tokens:
- if _DATE_RE.match(tok) and current:
- groups.append(current)
- current = [tok]
- else:
- current.append(tok)
- if current and current[0]:
- groups.append(current)
-
- new_row = [""] * len(new_header)
- for i, grp in enumerate(groups):
- if i >= n_groups:
- break
- new_row[i * 3] = grp[0]
- new_row[i * 3 + 1] = " ".join(grp[1:]).strip()
- new_row[i * 3 + 2] = amounts[i] if i < len(amounts) else ""
-
- new_rows.append(new_row)
-
- return new_rows
-
-_JB_SECTION_HDR = re.compile(
- r"^(Deposits?|Withdrawals?|Checks?(?:\s+Paid)?|Daily\s+Account\s+Balance)"
- r"\s*(?:\(cont\.?\))?\s*$",
- re.IGNORECASE,
-)
-_JB_COL_HDR = re.compile(
- r"^Date\s+(Description|Number)\s+Amount\s*$"
- r"|^Date\s+(?:Number\s+)?(?:Balance|Amount)\s*$",
- re.IGNORECASE,
-)
-_JB_DATE_DD = re.compile(r"^\d{2}-\d{2}\s+")
-_JB_TXN_ROW = re.compile(r"^(\d{2}-\d{2})\s+(.+?)\s+(-?\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
-_JB_BAL_ROW = re.compile(r"^(\d{2}-\d{2})\s+(\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
-
-def _jb_rows_to_html(header_cols, rows):
- th = "".join("
" + h + " | " for h in header_cols)
- parts = ["
", "" + th + "
"]
- for row in rows:
- td = "".join("" + str(row.get(h, "")) + " | " for h in header_cols)
- parts.append("" + td + "
")
- parts.append("
")
- return "\n".join(parts)
-
-def _jb_section_has_table(lines, start, end):
- return any("
str:
- lines = text.splitlines()
- out = []
- i = 0
-
- while i < len(lines):
- line = lines[i].strip()
-
- if not _JB_SECTION_HDR.match(line):
- out.append(lines[i])
- i += 1
- continue
-
- section_title = line
- j = i + 1
-
- while j < len(lines) and not lines[j].strip():
- j += 1
-
- if j >= len(lines) or not _JB_COL_HDR.match(lines[j].strip()):
- out.append(lines[i])
- i += 1
- continue
-
- col_hdr_line = lines[j].strip()
- data_start = j + 1
-
- first_data_idx = _find_first_data_row(lines, data_start)
-
- if first_data_idx is None or not _JB_DATE_DD.match(lines[first_data_idx].strip()):
- out.append(lines[i])
- i += 1
- continue
-
- section_end = len(lines)
- for k in range(data_start, len(lines)):
- l = lines[k].strip()
- if (l != section_title and _JB_SECTION_HDR.match(l)
- or l.startswith("---page-separator")
- or l.startswith("Page:")):
- section_end = k
- break
-
- if _jb_section_has_table(lines, i, section_end):
- out.append(lines[i])
- i += 1
- continue
-
- parts_upper = col_hdr_line.upper().split()
- if len(parts_upper) == 2 and parts_upper[1] == "BALANCE":
- header_cols = ["Date", "Balance"]
- elif "NUMBER" in parts_upper:
- header_cols = ["Date", "Number", "Amount"]
- else:
- header_cols = ["Date", "Description", "Amount"]
-
- out.append(section_title)
- rows = []
- current = None
- k = data_start
-
- while k < section_end:
- raw = lines[k]
- row_line = raw.strip()
-
- if not row_line:
- k += 1
- continue
-
- if (_JB_SECTION_HDR.match(row_line)
- or row_line.startswith("---page-separator")
- or row_line.startswith("Page:")):
- break
-
- mb = _JB_BAL_ROW.match(row_line)
- if mb and header_cols == ["Date", "Balance"]:
- if current:
- rows.append(current)
- current = {"Date": mb.group(1), "Balance": mb.group(2)}
- k += 1
- continue
-
- mt = _JB_TXN_ROW.match(row_line)
- if mt:
- if current:
- rows.append(current)
- desc_key = "Description" if "Description" in header_cols else "Number"
- current = {
- "Date": mt.group(1),
- desc_key: mt.group(2).strip(),
- "Amount": mt.group(3),
- }
- k += 1
- continue
-
- if current and row_line:
- desc_key = "Description" if "Description" in header_cols else "Number"
- if desc_key in current:
- current[desc_key] += " " + row_line
- k += 1
- continue
-
- k += 1
-
- if current:
- rows.append(current)
-
- if rows:
- out.append(_jb_rows_to_html(header_cols, rows))
-
- i = section_end
- continue
-
- return "\n".join(out)
-
-# --------------------------
-# Navy Federal full-page text-layer parser (Fix NF-1)
-# --------------------------
-
-def _is_nfcu_document(pdf_path: str) -> bool:
- try:
- import pymupdf as fitz
- doc = fitz.open(pdf_path)
- pages_to_check = min(2, len(doc))
- text = ""
- for i in range(pages_to_check):
- text += "\n" + (doc[i].get_text() or "")
- doc.close()
- t = text.lower()
- return (
- ("statement of account" in t)
- and ("access no." in t)
- and ("routing number" in t)
- and ("navy federal" in t or "navyfederal.org" in t)
- )
- except Exception:
- return False
-
-
-def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
- try:
- import pymupdf as fitz
-
- doc = fitz.open(pdf_path)
- if page_num >= len(doc):
- doc.close()
- return ""
- page = doc[page_num]
- page_text = page.get_text() or ""
- words = page.get_text("words") or []
- doc.close()
-
- if not page_text or not words:
- return ""
-
- t = page_text.lower()
- is_nfcu = (
- ("statement of account" in t)
- and (("access no." in t) or ("routing number" in t) or ("navy federal" in t) or ("navyfederal.org" in t))
+ with gr.Blocks(title="GLM-OCR (simple)") as demo:
+ gr.Markdown(
+ "# GLM-OCR (simple)\n"
+ "Upload a PDF or image. Header and footer bands are included; "
+ "body OCR is passed through with only light markdown cleanup."
)
- if not is_nfcu:
- return ""
-
- buckets = []
- for w in words:
- if len(w) < 5:
- continue
- x0 = float(w[0])
- y0 = float(w[1])
- token = w[4].strip()
- if not token:
- continue
- bi = None
- for i, (yb, _) in enumerate(buckets):
- if abs(yb - y0) <= 2.0:
- bi = i
- break
- if bi is None:
- buckets.append((y0, [(x0, token)]))
- else:
- buckets[bi][1].append((x0, token))
-
- lines = []
- for y, arr in sorted(buckets, key=lambda x: x[0]):
- toks = [tok for _, tok in sorted(arr, key=lambda x: x[0])]
- xs = [x for x, _ in sorted(arr, key=lambda x: x[0])]
- lines.append((y, xs, toks, " ".join(toks)))
-
- date_re = re.compile(r"^\d{2}-\d{2}$")
- money_re = re.compile(r"^[\d,]+\.\d{2}-?$")
- acct_re = re.compile(r"^(Business Checking|Mbr Business Savings)\s*-\s*(\d+)")
- stop_markers = (
- "Items Paid",
- "Average Daily Balance",
- "2024 Year to Date",
- "Disclosure",
- "What to Do",
- "Errors",
- "Payments",
- "SAVINGS DIVIDENDS",
- "REMITTANCE RECEIVED",
- "DEPOSIT VOUCHER",
- "ACCOUNT NUMBER",
- "MARK \"X\"",
- "ADDRESS/ORDER",
- "ITEMS ON REVERSE",
- "CHANGE OF ADDRESS",
- "PLEASE PRINT",
- "RANK/RATE",
- "PO BOX",
- "MERRIFIELD",
- "Questions about this Statement",
+ file_in = gr.File(
+ label="Upload PDF or image",
+ file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"],
)
-
- def parse_account_rows(start_idx):
- rows = []
- i = start_idx
- pending = None
- while i < len(lines):
- _, _, toks, text = lines[i]
- if not text:
- i += 1
- continue
-
- if acct_re.match(text) and i != start_idx:
- break
- if any(text.startswith(s) for s in stop_markers):
- break
- upper_text = text.upper()
- if (
- "REMITTANCE RECEIVED" in upper_text
- or "DEPOSIT VOUCHER" in upper_text
- or "ACCOUNT NUMBER" in upper_text
- or "ADDRESS/ORDER" in upper_text
- or "ITEMS ON REVERSE" in upper_text
- or "CHANGE OF ADDRESS" in upper_text
- or "PLEASE PRINT" in upper_text
- ):
- break
- if text in ("Checking", "Savings"):
- break
- if text.startswith("Date Transaction Detail"):
- i += 1
- continue
-
- def flush_pending():
- nonlocal pending
- if pending is not None:
- rows.append(pending)
- pending = None
-
- if toks and date_re.match(toks[0]):
- flush_pending()
- date = toks[0]
- rest = toks[1:]
- mvals = [tok for tok in rest if money_re.match(tok)]
- desc_tokens = [tok for tok in rest if not money_re.match(tok)]
-
- amount = ""
- balance = ""
- if len(mvals) >= 2:
- amount = mvals[-2]
- balance = mvals[-1]
- elif len(mvals) == 1:
- balance = mvals[-1]
-
- pending = {
- "date": date,
- "desc": " ".join(desc_tokens).strip(),
- "amount": amount,
- "balance": balance,
- }
- else:
- if pending is not None:
- mvals = [tok for tok in toks if money_re.match(tok)]
- desc_tokens = [tok for tok in toks if not money_re.match(tok)]
-
- upper = text.upper()
- if pending["balance"] and (
- "REMITTANCE" in upper
- or "DEPOSIT VOUCHER" in upper
- or "ACCOUNT NUMBER" in upper
- or "ADDRESS/ORDER" in upper
- or "ITEMS ON REVERSE" in upper
- or "PO BOX" in upper
- ):
- break
-
- if (
- ("BEGINNING BALANCE" in pending["desc"].upper() or "ENDING BALANCE" in pending["desc"].upper())
- and not mvals
- and not (toks and date_re.match(toks[0]))
- ):
- break
- if desc_tokens:
- pending["desc"] = (pending["desc"] + " " + " ".join(desc_tokens)).strip()
- if len(mvals) >= 2 and (not pending["amount"] or not pending["balance"]):
- if not pending["amount"]:
- pending["amount"] = mvals[-2]
- if not pending["balance"]:
- pending["balance"] = mvals[-1]
- elif len(mvals) == 1 and not pending["balance"]:
- pending["balance"] = mvals[-1]
- i += 1
-
- if pending is not None:
- rows.append(pending)
- cleaned = [r for r in rows if r["date"] and r["desc"] and (r["amount"] or r["balance"])]
- return cleaned, i
-
- out = []
- i = 0
- while i < len(lines):
- _, _, _, text = lines[i]
- if not text:
- i += 1
- continue
-
- m = acct_re.match(text)
- if m:
- acct_name = m.group(1)
- acct_no = m.group(2)
- title = f"{acct_name} - {acct_no}"
- if i + 1 < len(lines) and "Continued" in lines[i + 1][3]:
- title += " (Continued from previous page)"
- out.append(f"{title}")
- out.append("")
- out.append("| Date | Transaction Detail | Amount($) | Balance($) |
|---|
")
- rows, i2 = parse_account_rows(i + 1)
- for r in rows:
- out.append(
- f"| {r['date']} | {r['desc']} | {r['amount']} | {r['balance']} |
"
- )
- out.append("
")
- i = i2
- continue
-
- if text in ("Checking", "Savings"):
- out.append(f"{text}
")
- elif text.startswith("Items Paid"):
- out.append("Items Paid")
- i += 1
-
- return "\n".join(out).strip()
-
- except Exception as e:
- log.warning("_parse_nfcu_page failed (page %d): %s", page_num, e)
- return ""
-
-def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
- try:
- import pymupdf as fitz
-
- doc = fitz.open(pdf_path)
- if page_num >= len(doc):
- doc.close()
- return ""
- page = doc[page_num]
- words = page.get_text("words") or []
- doc.close()
- if not words:
- return ""
-
- buckets = []
- for w in words:
- if len(w) < 5:
- continue
- x0 = float(w[0])
- y0 = float(w[1])
- tok = str(w[4] or "").strip()
- if not tok:
- continue
- bi = None
- for i, (yb, _) in enumerate(buckets):
- if abs(yb - y0) <= 2.0:
- bi = i
- break
- if bi is None:
- buckets.append((y0, [(x0, tok)]))
- else:
- buckets[bi][1].append((x0, tok))
-
- lines = []
- for y, arr in sorted(buckets, key=lambda x: x[0]):
- arr2 = sorted(arr, key=lambda x: x[0])
- toks = [t for _, t in arr2]
- text = " ".join(toks).strip()
- lines.append((y, toks, text))
-
- anchor_idx = None
- for i, (_, _, text) in enumerate(lines):
- tl = text.lower()
- if "summary of your deposit accounts" in tl or ("summary" in tl and "deposit accounts" in tl):
- anchor_idx = i
- break
- if anchor_idx is None:
- return ""
-
- money_re = re.compile(r"^\$?[\d,]+\.\d{2}-?$")
- acct_no_re = re.compile(r"^\d{8,12}$")
- stop_re = re.compile(
- r"(business checking\s*-|mbr business savings\s*-|date\s+transaction\s+detail|items paid)",
- re.IGNORECASE,
- )
-
- rows = []
- pending_account = ""
- account_name_re = re.compile(
- r"^(business checking|mbr business savings|checking|savings|money market|certificate).*$",
- re.IGNORECASE,
- )
-
- for _, toks, text in lines[anchor_idx + 1:]:
- if not text:
- continue
- if stop_re.search(text):
- break
- if not toks:
- continue
- t0 = toks[0].lower()
- tl = text.lower()
-
- if (
- "previous" in tl
- or "deposits" in tl
- or "withdrawals" in tl
- or "ending" in tl
- or "dividends" in tl
- ) and not any(money_re.match(t) for t in toks):
- continue
-
- if account_name_re.match(text.strip()) and not any(money_re.match(t) for t in toks):
- pending_account = text.strip()
- continue
-
- mvals = [t.replace("$", "") for t in toks if money_re.match(t)]
- acct_tokens = [t for t in toks if acct_no_re.match(t)]
-
- if t0 == "totals" and len(mvals) >= 5:
- rows.append(["Totals", mvals[0], mvals[1], mvals[2], mvals[3], mvals[4]])
- continue
-
- if acct_tokens and len(mvals) >= 5:
- acct = acct_tokens[0]
- acct_label = (pending_account + " - " + acct).strip(" -") if pending_account else acct
- rows.append([acct_label, mvals[0], mvals[1], mvals[2], mvals[3], mvals[4]])
- pending_account = ""
- continue
-
- if not rows:
- return ""
-
- hdr = [
- "Account",
- "Previous Balance",
- "Deposits/Credits",
- "Withdrawals/Debits",
- "Ending Balance",
- "YTD Dividends",
- ]
- return _grid_to_html([hdr] + rows)
- except Exception:
- return ""
-
-def _replace_nfcu_summary_table(page_md: str, summary_table_html: str) -> str:
- if not page_md or not summary_table_html:
- return page_md
- table_pattern = re.compile(r"]*>.*?