fragments
+ (e.g. 2293 $6,721.00 4/29/2025). Collapse detected gallery pages into one Check #|Amount|Date
+ table when bbox/image markers + repeated check lines are present; skips chunks with YOUR CHECKS SEQUENCED.
+- Fix MD-4: YOUR CHECKS SEQUENCED wide grid (N×DATE|CHECK#|AMOUNT per row) is unfolded to one DATE|CHECK#|AMOUNT
+ table so narrow viewers show every check, not only the first date column.
+- Fix FH-1: Drop a leading Date|Description|Amount (DA3) table when its (date, amount) multiset is a
+ strict subset of a later DEPOSIT/WITHDRAWAL register (e.g. First Horizon OCR duplicate before
+ ACCOUNT SUMMARY). Safe no-op when no such pair exists.
+- Fix FH-2: Normalize packed DAILY BALANCE SUMMARY and CHECKS PAID SUMMARY tables into one-row-per-item
+ layouts; also promote multiline register headers where row0 is placeholder and row1 has true
+ DATE/DESCRIPTION/DEPOSIT/WITHDRAWAL labels.
+- Fix SE-multi: Multi-account deposit summary tables (Beginning/Deposits/Withdrawals/Ending) sometimes get a
+ bogus OCR sub-row with tiny amounts (e.g. -$1 / $2 / $1) next to a real account row. Drop those
+ dust rows when another row has statement-scale balances, then recompute the TOTAL row from the
+ remaining account rows. Generic; no institution names.
+"""
+
+# Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
+import asyncio
+try:
+ _orig_close = asyncio.BaseEventLoop.close
+ def _safe_close(self):
+ try:
+ _orig_close(self)
+ except (ValueError, OSError):
+ pass
+ asyncio.BaseEventLoop.close = _safe_close
+except Exception:
+ pass
import logging
import os
import re
+import html
import tempfile
-import uuid
-from typing import List, Tuple
+from typing import List, Optional, Tuple
+from collections import Counter, defaultdict
+from html.parser import HTMLParser
+
+import yaml
+
+# Gradio and glmocr are optional at import: pro_processor only needs stabilize_tables_and_text
+# (glm_ocr_client). HF Space / run_ocr still use glmocr + gradio when installed.
+try:
+ import glmocr
+
+ GLMOCR_BASE = os.path.dirname(glmocr.__file__)
+ CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
+ FORMATTER_PATH = os.path.join(GLMOCR_BASE, "postprocess", "result_formatter.py")
+except ImportError:
+ glmocr = None # type: ignore
+ GLMOCR_BASE = ""
+ CONFIG_PATH = ""
+ FORMATTER_PATH = ""
+
+log = logging.getLogger("glmocr_app")
+logging.basicConfig(level=logging.INFO)
+
+# ============================================================
+# HARD-CODED SETTINGS (edit these numbers to tune quality/speed)
+# ============================================================
+
+GLMOCR_API_KEY = "cee1d52dd91a4ab591b3f6e105f8ad89.LgbQTECuzX0zrito"
+
+# Higher = better OCR for small/right-aligned digits; slower
+RENDER_SCALE = 2.2 # try 2.5 if right-side numbers are missed
+
+# Padding to protect columns near edges (Balance is often right-most)
+PAD_LEFT_FRAC = 0.02
+PAD_RIGHT_FRAC = 0.06 # try 0.10 if right-most balances are missing
+PAD_TOP_FRAC = 0.01
+PAD_BOTTOM_FRAC = 0.01
+
+ENABLE_CONTRAST = True
+
+DEFAULT_ZONE_FRAC = 0.12
+PDF_HEADER_BAND_FRAC = 0.10
+
+ENABLE_FOOTER_OCR = True # enabled — 3-step fallback with dedup guard
+PDF_FOOTER_BAND_FRAC = 0.88
+
+MIN_CROP_HEIGHT = 112
+MIN_CROP_PIXELS = 112 * 112
+
+# ============================================================
+
+_parser = None
+
+def get_parser():
+ global _parser
+ if _parser is None:
+ from glmocr import GlmOcr
+ _parser = GlmOcr(api_key=GLMOCR_API_KEY, mode="maas")
+ return _parser
+
+# ---------------------------------------------------------------------------
+# Best-effort config tweaks (safe to fail on read-only HF env)
+# ---------------------------------------------------------------------------
+if CONFIG_PATH:
+ try:
+ with open(CONFIG_PATH, "r") as f:
+ config = yaml.safe_load(f)
+ config["pipeline"]["maas"]["enabled"] = True
+ config["pipeline"]["maas"]["api_key"] = GLMOCR_API_KEY
+ with open(CONFIG_PATH, "w") as f:
+ yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+ except Exception:
+ pass
+
+# Best-effort formatter tweak: avoid stripping header/footer labels
+if FORMATTER_PATH:
+ try:
+ with open(FORMATTER_PATH, "r") as f:
+ source = f.read()
+ for label in (
+ '"header"', "'header'",
+ '"footer"', "'footer'",
+ '"doc_header"', "'doc_header'",
+ '"doc_footer"', "'doc_footer'",
+ ):
+ source = re.sub(r",\s*" + re.escape(label), "", source)
+ source = re.sub(re.escape(label) + r"\s*,", "", source)
+ source = re.sub(re.escape(label), "", source)
+ with open(FORMATTER_PATH, "w") as f:
+ f.write(source)
+ except Exception:
+ pass
+
+# --------------------------
+# Header/footer helpers
+# --------------------------
+def get_header_footer_zones(regions, norm_height=1000):
+ if not regions:
+ return None, None
+ y_tops, y_bottoms = [], []
+ for r in regions:
+ bbox = r.get("bbox_2d") if isinstance(r, dict) else getattr(r, "bbox_2d", None)
+ if bbox and len(bbox) >= 4:
+ y_tops.append(bbox[1])
+ y_bottoms.append(bbox[3])
+ if not y_tops:
+ return None, None
+ return min(y_tops) / norm_height, max(y_bottoms) / norm_height
+
+def extract_zone_text_pdf(pdf_path, page_num, y_start_frac, y_end_frac):
+ try:
+ import pymupdf as fitz
+ doc = fitz.open(pdf_path)
+ page = doc[page_num]
+ h, w = page.rect.height, page.rect.width
+ rect = fitz.Rect(0, h * y_start_frac, w, h * y_end_frac)
+ text = page.get_text(clip=rect).strip()
+ doc.close()
+ return text
+ except Exception:
+ return ""
+
+def extract_pdf_text_in_band(pdf_path, page_num, y_start_frac, y_end_frac):
+ try:
+ import pymupdf as fitz
+ doc = fitz.open(pdf_path)
+ page = doc[page_num]
+ h = page.rect.height
+ y_lo = h * y_start_frac
+ y_hi = h * y_end_frac
+ words = page.get_text("words")
+ doc.close()
+ parts = []
+ for w in words:
+ if len(w) >= 5:
+ y0, y1 = float(w[1]), float(w[3])
+ if y0 < y_hi and y1 > y_lo:
+ parts.append(w[4])
+ return " ".join(parts).strip()
+ except Exception:
+ return ""
+
+def ocr_zone(image_path, y_start_frac, y_end_frac):
+ zone_name = "header" if y_end_frac < 0.5 else "footer"
+ try:
+ from PIL import Image
+ img = Image.open(image_path).convert("RGB")
+ w, h = img.size
+ y0 = max(0, int(h * y_start_frac))
+ y1 = min(h, int(h * y_end_frac))
+ if y1 <= y0:
+ return ""
+
+ crop = img.crop((0, y0, w, y1))
+ cw, ch = crop.size
+
+ if ch < MIN_CROP_HEIGHT or (cw * ch) < MIN_CROP_PIXELS:
+ need_h = max(ch, MIN_CROP_HEIGHT)
+ need_w = max(cw, 1)
+ if (need_w * need_h) < MIN_CROP_PIXELS:
+ need_w = max(need_w, (MIN_CROP_PIXELS + need_h - 1) // need_h)
+ canvas = Image.new("RGB", (need_w, need_h), (255, 255, 255))
+ if zone_name == "header":
+ canvas.paste(crop, (0, 0))
+ else:
+ canvas.paste(crop, (0, need_h - ch))
+ crop = canvas
+
+ fd, path = tempfile.mkstemp(suffix=".jpg")
+ os.close(fd)
+ try:
+ crop.save(path, "JPEG", quality=92)
+ parser = get_parser()
+ out = parser.parse(path)
+ if not isinstance(out, list):
+ out = [out]
+ if out and getattr(out[0], "markdown_result", None):
+ return (out[0].markdown_result or "").strip()
+ finally:
+ try:
+ os.unlink(path)
+ except Exception:
+ pass
+ except Exception as e:
+ log.warning("[%s] ocr_zone failed: %s", zone_name, e, exc_info=True)
+ return ""
+
+def fix_account_number(hdr: str) -> str:
+ if not hdr:
+ return hdr
+ if "Account Number:" in hdr and "Account Number: " not in hdr:
+ m = re.search(r"[0-9]{5,}", hdr)
+ if m:
+ hdr = hdr.replace("Account Number:", "Account Number: " + m.group(0))
+ acct_match = re.search(r"Account Number: ([0-9]{5,})", hdr)
+ if acct_match:
+ acct = acct_match.group(1)
+ if hdr.startswith(acct):
+ hdr = hdr[len(acct):].lstrip()
+ return hdr
+
+# --------------------------
+# Table stabilization helpers
+# --------------------------
+def close_unclosed_html(md: str) -> str:
+ if not md:
+ return md
+ open_tags = re.findall(r"<(table|tbody|thead|tr|td|th)\b", md, flags=re.IGNORECASE)
+ close_tags = re.findall(r"", md, flags=re.IGNORECASE)
+
+ def count(tags, name):
+ return sum(1 for t in tags if t.lower() == name)
+
+ for tag in reversed(["td", "th", "tr", "thead", "tbody", "table"]):
+ opened = count(open_tags, tag)
+ closed = count(close_tags, tag)
+ if opened > closed:
+ md += ("" % tag) * (opened - closed)
+ return md
+
+def looks_like_markdown_table(block: str) -> bool:
+ lines = [ln.rstrip() for ln in block.strip().splitlines() if ln.strip()]
+ if len(lines) < 2:
+ return False
+ if "|" not in lines[0]:
+ return False
+ sep = lines[1].replace(" ", "")
+ return ("---" in sep) and ("|" in sep)
+
+def md_table_to_html(block: str) -> str:
+ lines = [ln.strip() for ln in block.strip().splitlines() if ln.strip()]
+ if len(lines) < 2:
+ return block
+
+ def split_row(row: str):
+ row = row.strip()
+ if row.startswith("|"):
+ row = row[1:]
+ if row.endswith("|"):
+ row = row[:-1]
+ return [p.strip() for p in row.split("|")]
+
+ header = split_row(lines[0])
+ body_lines = [ln for ln in lines[2:] if "|" in ln]
+
+ html_rows = []
+ html_rows.append("
" + "".join(f"| {html.escape(c)} | " for c in header) + "
")
+ for ln in body_lines:
+ cols = split_row(ln)
+ if len(cols) < len(header):
+ cols += [""] * (len(header) - len(cols))
+ html_rows.append("
" + "".join(f"| {html.escape(c)} | " for c in cols[: len(header)]) + "
")
+ return "
\n" + "\n".join(html_rows) + "\n
"
+
+def normalize_money_glyphs(text: str) -> str:
+ if not text:
+ return text
+ t = text.replace("−", "-").replace("–", "-").replace("—", "-")
+ t = re.sub(
+ r"\(\s*\$?\s*([0-9]{1,3}(?:,[0-9]{3})*|[0-9]+)(\.[0-9]{2})\s*\)",
+ r"-\1\2",
+ t,
+ )
+
+ def o_to_zero(m):
+ token = m.group(0)
+ return token.replace("O", "0").replace("o", "0")
+
+ t = re.sub(r"\b[0-9Oo\$,.\-]{4,}\b", o_to_zero, t)
+ return t
+
+# ---- Generic HTML table normalizer ----
+class TableGridParser(HTMLParser):
+ """Parse a
into rows of (text, colspan, rowspan)."""
+
+ def __init__(self):
+ super().__init__()
+ self.rows = []
+ self._current_row = []
+ self._cell_text = []
+ self._colspan = 1
+ self._rowspan = 1
+ self._in_cell = False
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "tr":
+ self._current_row = []
+ elif tag in ("td", "th"):
+ attrs_d = dict(attrs)
+ self._colspan = max(1, int(attrs_d.get("colspan", 1)))
+ self._rowspan = max(1, int(attrs_d.get("rowspan", 1)))
+ self._cell_text = []
+ self._in_cell = True
+
+ def handle_endtag(self, tag):
+ if tag in ("td", "th"):
+ text = "".join(self._cell_text).strip().replace("\n", " ")
+ self._current_row.append((text, self._colspan, self._rowspan))
+ self._in_cell = False
+ elif tag == "tr":
+ self.rows.append(self._current_row)
+
+ def handle_data(self, data):
+ if self._in_cell:
+ self._cell_text.append(data)
+
+def _build_grid(rows_data):
+ if not rows_data:
+ return []
+ blocked = defaultdict(set)
+ grid = []
+
+ for r, row_cells in enumerate(rows_data):
+ grid.append([])
+ col = 0
+ for content, C, R in row_cells:
+ while col in blocked[r]:
+ grid[r].append("")
+ col += 1
+ for k in range(C):
+ grid[r].append(content if k == 0 else "")
+ for k in range(1, R):
+ blocked[r + k].add(col)
+ col += C
+
+ max_cols = max(len(row) for row in grid) if grid else 0
+ for row in grid:
+ while len(row) < max_cols:
+ row.append("")
+ return grid
+
+def _grid_to_html(grid):
+ if not grid:
+ return ""
+ lines = [""]
+ for r, row in enumerate(grid):
+ lines.append("")
+ tag = "th" if r == 0 else "td"
+ for cell in row:
+ escaped = (
+ (cell or "")
+ .replace("&", "&")
+ .replace("<", "<")
+ .replace(">", ">")
+ .replace('"', """)
+ )
+ lines.append(f"<{tag}>{escaped}")
+ lines.append("
")
+ lines.append("
")
+ return "\n".join(lines)
+
+def _is_amount_like(s: str) -> bool:
+ if not s:
+ return False
+ return re.fullmatch(r"\$?-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?", s.strip()) is not None
+
+def _drop_truly_empty_columns(grid):
+ if not grid or len(grid) < 1:
+ return grid
+
+ header = [str(c or "").strip() for c in grid[0]]
+ ncols = len(header)
+ if ncols <= 1:
+ return grid
+
+ body = grid[1:]
+ keep = [True] * ncols
+
+ for i in range(ncols):
+ if header[i] != "":
+ continue
+ total = 0
+ non_empty = 0
+ for row in body:
+ if i >= len(row):
+ continue
+ total += 1
+ if str(row[i] or "").strip():
+ non_empty += 1
+ if total > 0 and (non_empty / total) <= 0.05:
+ keep[i] = False
+
+ if all(keep):
+ return grid
+
+ new_grid = []
+ for row in grid:
+ new_grid.append([cell for idx, cell in enumerate(row) if idx < len(keep) and keep[idx]])
+ return new_grid
+
+
+def _is_multigroup_check_list_header(header: List[str]) -> bool:
+ """
+ Wide check listings (e.g. YOUR CHECKS SEQUENCED) use one row with a title cell, repeated
+ Amount headers, and blank slots between DATE | CHECK # | AMOUNT triplets. Fix 3
+ blank-header merge would otherwise fuse the next column into Amount or the section title,
+ leaving only the first date column visibly populated.
+ """
+ if not header:
+ return False
+ joined = " ".join(str(c or "").strip() for c in header).upper()
+ if "SEQUENCED" in joined and "CHECK" in joined:
+ return True
+ # Repeated Amount columns in one header row = multi-group layout; do not merge blanks.
+ if sum(1 for c in header if str(c or "").strip().upper() == "AMOUNT") >= 2:
+ return True
+ return False
+
+
+def _should_skip_blank_header_merge_multiline_register(grid) -> bool:
+ """
+ Some banks (e.g. First Horizon) use a title row with blank | cells, then a second header
+ row with DATE DESCRIPTION | DEPOSIT | WITHDRAWAL | CARD #. _merge_blank_header_text_columns
+ would treat those blanks as OCR splits and merge columns — corrupting the grid.
+ """
+ if not grid or len(grid) < 2:
+ return False
+ h0_join = " ".join(str(c or "").strip() for c in grid[0]).upper()
+ if "ACCOUNT HISTORY" in h0_join:
+ return True
+ for ri in range(1, min(5, len(grid))):
+ cells = [str(c or "").strip().upper() for c in grid[ri]]
+ if not any(cells):
+ continue
+ row_join = " ".join(cells)
+ if "DEPOSIT" in row_join and "WITHDRAWAL" in row_join:
+ if "DATE" in row_join or "DESCRIPTION" in row_join:
+ return True
+ return False
+
+
+def _merge_blank_header_text_columns(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ header = [str(c or "").strip() for c in grid[0]]
+ if _is_multigroup_check_list_header(header):
+ return grid
+ if _should_skip_blank_header_merge_multiline_register(grid):
+ return grid
+
+ ncols = len(header)
+ body = grid[1:]
+ blank_cols = [i for i, h in enumerate(header) if h == ""]
+ if not blank_cols:
+ return grid
+
+ keep = [True] * ncols
+
+ for i in blank_cols:
+ j = i - 1
+ while j >= 0 and header[j] == "":
+ j -= 1
+ if j < 0:
+ continue
+
+ total = 0
+ non_empty = 0
+ texty = 0
+ for row in body:
+ if i >= len(row) or j >= len(row):
+ continue
+ v = str(row[i] or "").strip()
+ total += 1
+ if v:
+ non_empty += 1
+ if not _is_amount_like(v):
+ texty += 1
+
+ if total == 0:
+ continue
+ non_empty_ratio = non_empty / total
+ texty_ratio = (texty / non_empty) if non_empty else 0.0
+
+ if non_empty_ratio >= 0.55 and texty_ratio >= 0.70:
+ for r in range(1, len(grid)):
+ row = grid[r]
+ if i >= len(row) or j >= len(row):
+ continue
+ left = str(row[j] or "").strip()
+ right = str(row[i] or "").strip()
+ if right:
+ row[j] = (left + " " + right).strip() if left else right
+ keep[i] = False
+
+ if all(keep):
+ return grid
+
+ new_grid = []
+ for row in grid:
+ new_grid.append([cell for idx, cell in enumerate(row) if idx < len(keep) and keep[idx]])
+ return new_grid
+
+
+def _split_wide_checks_sequenced_grid(grid: List[List[str]]) -> Optional[List[List[str]]]:
+ """
+ Fix MD-4: Banks print YOUR CHECKS SEQUENCED as 2–3 side-by-side DATE|CHECK #|AMOUNT groups per row.
+ OCR HTML keeps that as 6–9 columns; narrow markdown previews only show the first date column.
+ Unfold into a single standard 3-column table (one row per check).
+ """
+ if not grid or len(grid) < 2:
+ return None
+ header = [str(c or "").strip() for c in grid[0]]
+ header_join = " ".join(header).upper()
+ if "SEQUENCED" not in header_join or "CHECK" not in header_join:
+ return None
+ ncols = max(len(r) for r in grid if isinstance(r, list))
+ if ncols < 6 or ncols % 3 != 0:
+ return None
+ amt_cols = sum(1 for c in header if str(c or "").strip().upper() == "AMOUNT")
+ if amt_cols < 2 and ncols < 9:
+ return None
+
+ date_re = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
+ out: List[List[str]] = [["DATE", "CHECK #", "AMOUNT"]]
+
+ for row in grid[1:]:
+ cells = [str(c or "").strip() for c in row]
+ while len(cells) < ncols:
+ cells.append("")
+ cells = cells[:ncols]
+ c0 = cells[0] if cells else ""
+ joined = " ".join(cells)
+
+ if c0.startswith("=") and ("=" * 3 in c0 or len(c0) >= 8):
+ continue
+ if c0 and "=" in c0 and not c0.replace("=", "").strip():
+ continue
+ if "DATE" in c0.upper() and "CHECK" in c0.upper() and "AMOUNT" in c0.upper():
+ continue
+ if "GAP IN CHECK" in joined.upper() and "SEQUENCE" in joined.upper():
+ continue
+
+ for g in range(0, ncols, 3):
+ if g + 2 >= len(cells):
+ break
+ d, ck, am = cells[g], cells[g + 1], cells[g + 2]
+ if not d and not ck and not am:
+ continue
+ if d and date_re.match(d):
+ out.append([d, ck, am])
+
+ if len(out) < 2:
+ return None
+ return out
+
+
+def _fill_headless_trailing_amount_headers(grid):
+ """
+ Fix 32a: PDF layouts often omit a header above the rightmost amount column; OCR leaves a
+ trailing empty | . Downstream parsers then mis-associate columns. Fill only when the
+ column is predominantly strict currency (generic, bank-agnostic).
+ """
+ if not grid or len(grid) < 2:
+ return grid
+
+ header = [str(c or "").strip() for c in grid[0]]
+ ncols = len(header)
+ if ncols < 2:
+ return grid
+
+ i = ncols - 1
+ while i >= 0 and header[i] == "":
+ i -= 1
+ empty_tail_indices = list(range(i + 1, ncols))
+ if not empty_tail_indices:
+ return grid
+
+ body = grid[1:]
+ new_header = list(header)
+
+ for j in empty_tail_indices:
+ total = 0
+ amt_hits = 0
+ for row in body:
+ if j >= len(row):
+ continue
+ total += 1
+ v = str(row[j] or "").strip()
+ if v and _is_amount_like(v):
+ amt_hits += 1
+ if total == 0:
+ continue
+ if (amt_hits / total) < 0.45:
+ continue
+
+ left = j - 1
+ while left >= 0 and not new_header[left]:
+ left -= 1
+ prev = (new_header[left] if left >= 0 else "").strip()
+ pl = prev.lower()
+ if any(
+ k in pl
+ for k in (
+ "addition",
+ "subtraction",
+ "credit",
+ "debit",
+ "deposit",
+ "withdrawal",
+ )
+ ):
+ new_header[j] = prev
+ else:
+ new_header[j] = "Amount"
+
+ grid[0] = new_header
+ return grid
+
+
+_MMDD_DATE_RE = re.compile(r"^\d{1,2}[-/]\d{2}(?:[-/]\d{2,4})?$")
+
+# Leading transaction-type cell when OCR splits category vs merchant (East West / similar).
+_STACKED_TXN_TYPE_RE = re.compile(
+ r"^(?:Pre-Auth Credit|Pre-Auth Debit|Preauth Debit|"
+ r"Withdrawal Trns|Withdrawal(?:\s|$)|Deposit(?:\s|$)|Deposit Transfe|"
+ r"Outgoing Wire|Incoming Wire|"
+ r"Service Charge|ACH Debit|ACH Credit|ACH Payment|ACH PMT|"
+ r"Wire Transfer|POS Purchase|ATM Withdrawal|Transfer Trns)\b",
+ re.IGNORECASE | re.DOTALL,
+)
+
+
+def _normalize_date_desc_money_table_alignment(grid):
+ """
+ Fix 32b: Align OCR tables with common bank PDF layouts.
+
+ * Credits (Additions): Number | Date | Transaction Description | Additions — Number is often
+ blank in the PDF but must exist as a column; OCR may omit it or split description across
+ columns and park amounts under a bogus \"Amount\" header.
+ * Debits (Subtractions): PDF is usually Date | Description | Subtractions (no Number) —
+ when OCR uses the same 4-column split error, merge back to three columns.
+
+ Handles (a) Date | short type | merchant tail | amount, (b) Date | full desc | empty | amount,
+ (c) continuation rows with a blank date. Per-row merge (no single global mode) so mixed OCR
+ errors (e.g. wrapped merchant lines) still normalize.
+ """
+ if not grid or len(grid) < 3:
+ return grid
+
+ header = [str(c or "").strip() for c in grid[0]]
+ n = len(header)
+
+ def _pad_row(cells, width):
+ cells = [str(c or "").strip() for c in cells]
+ while len(cells) < width:
+ cells.append("")
+ return cells[:width]
+
+ h0n = re.sub(r"\s+", " ", header[0]).strip().lower()
+ h1n = re.sub(r"\s+", " ", header[1]).strip().lower() if n > 1 else ""
+ h2n = re.sub(r"\s+", " ", header[2]).strip().lower() if n > 2 else ""
+ h3n = re.sub(r"\s+", " ", header[3]).strip().lower() if n > 3 else ""
+
+ money_label = header[2].strip() if n > 2 else ""
+ want_number_col = "addition" in h2n
+
+ # --- 3-column Date | Transaction Description | Additions/Subtractions: prepend Number for credits only ---
+ if n == 3:
+ if h0n != "date" or "transaction" not in h1n or "description" not in h1n:
+ return grid
+ if not any(
+ k in h2n
+ for k in (
+ "addition",
+ "subtraction",
+ "credit",
+ "debit",
+ "deposit",
+ "withdrawal",
+ )
+ ):
+ return grid
+ if not want_number_col:
+ return grid
+ out = [["Number", header[0], header[1], money_label or "Additions"]]
+ for row in grid[1:]:
+ c = _pad_row(row, 3)
+ if not any(c):
+ out.append(["", "", "", ""])
+ continue
+ out.append(["", c[0], c[1], c[2]])
+ return out
+
+ if n != 4:
+ return grid
+
+ if h0n != "date":
+ return grid
+ if "transaction" not in h1n or "description" not in h1n:
+ return grid
+ if not any(
+ k in h2n
+ for k in (
+ "addition",
+ "subtraction",
+ "credit",
+ "debit",
+ "deposit",
+ "withdrawal",
+ )
+ ):
+ return grid
+ if h3n not in ("", "amount") and not any(
+ k in h3n for k in ("addition", "subtraction", "credit", "debit")
+ ):
+ return grid
+
+ data_rows = [r for r in grid[1:] if any(str(c or "").strip() for c in r)]
+ if not data_rows:
+ return grid
+
+ has_date_led = False
+ for row in data_rows:
+ c = _pad_row(row, 4)
+ if c[0] and _MMDD_DATE_RE.match(c[0]):
+ has_date_led = True
+ break
+ if not has_date_led:
+ return grid
+
+ def _merge_date_led_row(c0, c1, c2, c3):
+ split_ok = (
+ bool(c1)
+ and bool(_STACKED_TXN_TYPE_RE.match(c1))
+ and bool(c2)
+ and not _is_amount_like(c2)
+ and _is_amount_like(c3)
+ )
+ if split_ok:
+ return (c1 + " " + c2).strip(), c3
+ shift_ok = (
+ bool(c1)
+ and not _is_amount_like(c1)
+ and not (c2 or "").strip()
+ and _is_amount_like(c3)
+ )
+ if shift_ok:
+ return c1, c3
+ dep_ok = (
+ bool(c1)
+ and bool(_STACKED_TXN_TYPE_RE.match(c1))
+ and not (c2 or "").strip()
+ and _is_amount_like(c3)
+ )
+ if dep_ok:
+ return c1, c3
+ desc = ((c1 + " " + c2).strip() or c1).strip()
+ amt = c3 if _is_amount_like(c3) else ""
+ return desc, amt
+
+ ml = money_label or ("Additions" if want_number_col else "Subtractions")
+
+ if want_number_col:
+ out_h = ["Number", header[0], header[1], ml]
+ else:
+ out_h = [header[0], header[1], ml]
+
+ out = [out_h]
+ for row in grid[1:]:
+ c = _pad_row(row, 4)
+ c0, c1, c2, c3 = c[0], c[1], c[2], c[3]
+ if not any(c):
+ out.append([""] * len(out_h))
+ continue
+ if c0 and _MMDD_DATE_RE.match(c0):
+ desc, amt = _merge_date_led_row(c0, c1, c2, c3)
+ if want_number_col:
+ out.append(["", c0, desc, amt])
+ else:
+ out.append([c0, desc, amt])
+ continue
+ if want_number_col:
+ desc_t = (c1 + " " + c2).strip()
+ amt_t = c3 if _is_amount_like(c3) else ""
+ out.append(["", c0, desc_t, amt_t])
+ else:
+ desc_t = (c1 + " " + c2).strip()
+ amt_t = c3 if _is_amount_like(c3) else ""
+ out.append([c0, desc_t, amt_t])
+
+ return out
+
+
+# --------------------------
+# Shared keyword definitions
+# --------------------------
+
+_MONEY_RE = re.compile(
+ r"^(?:[\$£€¥]|Rs\.?|INR|PKR)?\s*-?\s?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$",
+ re.IGNORECASE,
+)
+
+_HEADER_KW_PATTERNS = [
+ r"DATE",
+ r"POSTING\s+DATE",
+ r"VALUE\s+DATE",
+ r"TXN\s+DATE",
+ r"TRANSACTION\s+DATE",
+ r"ENTRY\s+DATE",
+ r"EFFECTIVE\s+DATE",
+ r"TRANSACTION(?:\s+(?:ID|TYPE|NO|NUMBER))?",
+ r"TXN(?:\s+(?:ID|NO|TYPE))?",
+ r"REF(?:ERENCE)?(?:\s*(?:NO|NUM|NUMBER))?",
+ r"CHEQUE(?:\s*(?:NO|NUMBER))?",
+ r"CHQ(?:\s*(?:NO|NUMBER))?",
+ r"VOUCHER(?:\s*(?:NO|NUMBER))?",
+ r"SR\.?\s*NO\.?",
+ r"SERIAL(?:\s*(?:NO|NUMBER))?",
+ r"DESCRIPTION",
+ r"DETAILS?",
+ r"PARTICULARS?(?:\s+OF\s+TRANSACTION)?",
+ r"NARRATION",
+ r"REMARKS?",
+ r"NOTES?",
+ r"DEBIT",
+ r"DEBITS?",
+ r"DR\.?",
+ r"WITHDRAWALS?",
+ r"PAID\s+OUT",
+ r"MONEY\s+OUT",
+ r"CREDIT",
+ r"CREDITS?",
+ r"CR\.?",
+ r"DEPOSITS?",
+ r"PAID\s+IN",
+ r"MONEY\s+IN",
+ r"AMOUNT",
+ r"BALANCE",
+ r"BAL\.?",
+ r"RUNNING\s+BALANCE",
+ r"RUNNING\s+BAL\.?",
+ r"AVAILABLE\s+BALANCE",
+ r"AVAILABLE\s+BAL\.?",
+ r"AVAIL\.?\s+BAL\.?",
+ r"LEDGER\s+BALANCE",
+ r"LEDGER\s+BAL\.?",
+ r"CLOSING\s+BALANCE",
+ r"CLOSING\s+BAL\.?",
+ r"OPENING\s+BALANCE",
+ r"OPENING\s+BAL\.?",
+]
+
+_HEADER_KW_RES = [
+ re.compile(r"^(" + p + r")(.*)", re.IGNORECASE | re.DOTALL)
+ for p in _HEADER_KW_PATTERNS
+]
+
+_HEADER_KW_EXACT_RE = re.compile(
+ r"^(?:" + r"|".join(_HEADER_KW_PATTERNS) + r")$",
+ re.IGNORECASE,
+)
+
+_HEADER_ROW_KEYWORD_THRESHOLD = 0.5
+
+# --------------------------
+# Fix 1: Fused-header recovery
+# --------------------------
+
+def _split_keyword_remainder(cell_text: str):
+ s = cell_text.strip()
+ for pattern in _HEADER_KW_RES:
+ m = pattern.match(s)
+ if m:
+ keyword = m.group(1).strip()
+ remainder = m.group(2).strip()
+ return keyword, remainder
+ return s, ""
+
+def _extract_fused_header_artifacts(header_row):
+ if not header_row:
+ return list(header_row), None
+
+ ncols = len(header_row)
+ cleaned = list(header_row)
+ recovered = [""] * ncols
+
+ text_fused = []
+ money_fused = []
+
+ for idx, cell in enumerate(header_row):
+ cell_s = str(cell or "").strip()
+ if not cell_s:
+ continue
+
+ keyword, remainder = _split_keyword_remainder(cell_s)
+
+ if not remainder:
+ continue
+
+ remainder_no_space = remainder.replace(" ", "")
+
+ if _MONEY_RE.match(remainder_no_space):
+ cleaned[idx] = keyword
+ recovered[idx] = remainder
+ money_fused.append(idx)
+
+ elif not _HEADER_KW_EXACT_RE.match(remainder.split()[0] if remainder.split() else ""):
+ cleaned[idx] = keyword
+ recovered[idx] = remainder
+ text_fused.append(idx)
+
+ if text_fused and money_fused:
+ return cleaned, recovered
+
+ return list(header_row), None
+
+def _clean_header_artifacts(grid):
+ if not grid or not grid[0]:
+ return grid, None
+
+ cleaned_header, recovered_row = _extract_fused_header_artifacts(grid[0])
+ grid[0] = cleaned_header
+ return grid, recovered_row
+
+# --------------------------
+# Fix 2: Misplaced header row promotion
+# --------------------------
+
+def _row_keyword_score(row):
+ non_empty = [str(c or "").strip() for c in row if str(c or "").strip()]
+ if not non_empty:
+ return 0.0
+
+ keyword_hits = 0
+ for cell in non_empty:
+ if _HEADER_KW_EXACT_RE.match(cell):
+ keyword_hits += 1
+ continue
+ parts = cell.split()
+ if all(_HEADER_KW_EXACT_RE.match(p) for p in parts) and len(parts) > 1:
+ keyword_hits += 1
+ continue
+
+ return keyword_hits / len(non_empty)
+
+_BALANCE_INFO_RE = re.compile(
+ r"(beginning|ending|opening|closing)\s+balance"
+ r"|account\s*#?\s*\d{5,}"
+ r"|\b\d{7,}\b"
+ r"|\$\d{1,3}(?:,\d{3})*\.\d{2}",
+ re.IGNORECASE,
+)
+
+def _header_contains_balance_info(header_row):
+ for cell in header_row:
+ if _BALANCE_INFO_RE.search(str(cell or "")):
+ return True
+ return False
+
+def _promote_misplaced_header_row(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ current_header_score = _row_keyword_score(grid[0])
+ if current_header_score >= _HEADER_ROW_KEYWORD_THRESHOLD:
+ return grid
+
+ candidate_idx = None
+ candidate_score = 0.0
+ for i in range(1, min(len(grid), 6)):
+ score = _row_keyword_score(grid[i])
+ if score >= _HEADER_ROW_KEYWORD_THRESHOLD and score > candidate_score:
+ candidate_score = score
+ candidate_idx = i
+
+ if candidate_idx is None:
+ return grid
+
+ candidate_row = grid[candidate_idx]
+
+ expanded_header = []
+ for cell in candidate_row:
+ cell_s = str(cell or "").strip()
+ parts = cell_s.split()
+ if len(parts) > 1 and all(_HEADER_KW_EXACT_RE.match(p) for p in parts):
+ expanded_header.extend(parts)
+ else:
+ expanded_header.append(cell_s)
+
+ new_ncols = len(expanded_header)
+ col_expansion = new_ncols - len(candidate_row)
+
+ metadata_row = None
+ if _header_contains_balance_info(grid[0]):
+ meta_cells = [str(c or "").strip() for c in grid[0]]
+ meta_text = " ".join(c for c in meta_cells if c).strip()
+ if meta_text:
+ metadata_row = [meta_text] + [""] * (new_ncols - 1)
+
+ new_data_rows = []
+ for row in grid[candidate_idx + 1:]:
+ if col_expansion > 0 and row:
+ first_cell = str(row[0] or "").strip()
+ date_match = re.match(r"^(\d{1,2}/\d{1,2})\s+(.*)", first_cell, re.DOTALL)
+ if date_match and col_expansion == 1:
+ new_row = [date_match.group(1), date_match.group(2).strip()] + list(row[1:])
+ else:
+ new_row = list(row) + [""] * col_expansion
+ new_data_rows.append(new_row)
+ else:
+ new_data_rows.append(list(row))
+
+ def pad(row, n):
+ r = list(row)
+ while len(r) < n:
+ r.append("")
+ return r[:n]
+
+ new_grid = [pad(expanded_header, new_ncols)]
+ if metadata_row is not None:
+ new_grid.append(pad(metadata_row, new_ncols))
+ for row in new_data_rows:
+ new_grid.append(pad(row, new_ncols))
+
+ return new_grid
+
+# --------------------------
+# Fix 3: Fused key-value rows in summary sections
+# --------------------------
+
+_FUSED_KV_RE = re.compile(
+ r"^(.+?)(-?\d{1,3}(?:,\d{3})*(?:\.\d+)?%?)$",
+ re.DOTALL,
+)
+
+def _fix_fused_keyvalue_rows(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ _DOLLAR_AMOUNT_RE = re.compile(r"\$\s*\d{1,3}(?:,\d{3})*\.\d{2}")
+ _LONG_NUMBER_RE = re.compile(r"\d{7,}")
+
+ ncols = len(grid[0])
+ new_grid = [grid[0]]
+
+ for row in grid[1:]:
+ cells = [str(c or "").strip() for c in row]
+ if not cells:
+ new_grid.append(row)
+ continue
+
+ first = cells[0]
+ rest_empty = all(c == "" for c in cells[1:])
+
+ if not rest_empty or not first:
+ new_grid.append(row)
+ continue
+
+ if _DOLLAR_AMOUNT_RE.search(first):
+ new_grid.append(row)
+ continue
+
+ if _LONG_NUMBER_RE.search(first):
+ new_grid.append(row)
+ continue
+
+ if re.search(r"#\s*[\dX]", first):
+ new_grid.append(row)
+ continue
+
+ m = _FUSED_KV_RE.match(first)
+ if not m:
+ new_grid.append(row)
+ continue
+
+ label_raw = m.group(1)
+ value = m.group(2).strip()
+
+ if label_raw != label_raw.rstrip():
+ new_grid.append(row)
+ continue
+
+ label = label_raw.strip()
+
+ if not label or not value:
+ new_grid.append(row)
+ continue
+
+ new_row = [""] * ncols
+ new_row[0] = label
+ new_row[ncols - 1] = value
+ new_grid.append(new_row)
+
+ return new_grid
+
+# --------------------------
+# Fix 4: PDF text-layer row-count patch
+# --------------------------
+
+def _extract_textlayer_rows(pdf_path: str, page_num: int):
+ try:
+ import pymupdf as fitz
+
+ doc = fitz.open(pdf_path)
+ page = doc[page_num]
+ words = page.get_text("words")
+ doc.close()
+
+ if not words:
+ return []
+
+ lines_by_y = {}
+ for w in words:
+ x0, y0, word = float(w[0]), float(w[1]), w[4]
+ bucket = None
+ for existing_y in lines_by_y:
+ if abs(existing_y - y0) <= 5:
+ bucket = existing_y
+ break
+ if bucket is None:
+ bucket = y0
+ lines_by_y.setdefault(bucket, []).append((x0, word))
+
+ sorted_lines = []
+ for y in sorted(lines_by_y):
+ line_words = sorted(lines_by_y[y], key=lambda t: t[0])
+ sorted_lines.append([w for _, w in line_words])
+
+ _MONTHS = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
+ date_re = re.compile(
+ r"^\d{1,2}/\d{2}(?:/\d{2,4})?$"
+ r"|^\d{1,2}-\d{2}(?:-\d{2,4})?$"
+ r"|^\d{4}-\d{2}-\d{2}$"
+ r"|^" + _MONTHS + r"$",
+ re.IGNORECASE,
+ )
+ month_re = re.compile(r"^" + _MONTHS + r"$", re.IGNORECASE)
+ day_re = re.compile(r"^\d{1,2}$")
+
+ amount_re = re.compile(
+ r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
+ r"|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
+ r"|^-\s+\$\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
+ )
+
+ rows = []
+ for line in sorted_lines:
+ if len(line) < 2:
+ continue
+
+ date_str = None
+ desc_start = None
+
+ if date_re.match(line[0]):
+ if month_re.match(line[0]) and len(line) > 1 and day_re.match(line[1]):
+ date_str = line[0] + " " + line[1]
+ desc_start = 2
+ else:
+ date_str = line[0]
+ desc_start = 1
+ else:
+ continue
+
+ if desc_start is None or desc_start >= len(line):
+ continue
+
+ remaining = line[desc_start:]
+ post_date_str = ""
+ if remaining and month_re.match(remaining[0]):
+ if len(remaining) > 1 and day_re.match(remaining[1]):
+ post_date_str = remaining[0] + " " + remaining[1]
+ remaining = remaining[2:]
+ elif len(remaining) > 0:
+ remaining = remaining[1:]
+ elif remaining and date_re.match(remaining[0]) and not month_re.match(remaining[0]):
+ post_date_str = remaining[0]
+ remaining = remaining[1:]
+
+ if len(remaining) < 2:
+ continue
+
+ if (len(remaining) >= 2
+ and remaining[-2] == "-"
+ and remaining[-1].startswith("$")):
+ amount_candidate = remaining[-2] + " " + remaining[-1]
+ desc_tokens = remaining[:-2]
+ else:
+ amount_candidate = remaining[-1]
+ desc_tokens = remaining[:-1]
+
+ if not amount_re.match(amount_candidate):
+ continue
+
+ desc = " ".join(desc_tokens).strip()
+ if not desc:
+ continue
+
+ if not re.search(r"[A-Za-z]", desc):
+ continue
+
+ rows.append({"date": date_str, "post_date": post_date_str, "desc": desc, "amount": amount_candidate})
+
+ return rows if len(rows) >= 2 else []
+
+ except Exception as e:
+ log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
+ return []
+
+def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
+ if not text_layer:
+ return []
+ m = re.search(
+ r"Checks\s*\([^\n]*\n(.*?)(?=^Daily\s+Account\s+Balance\s*$)",
+ text_layer,
+ re.DOTALL | re.MULTILINE | re.IGNORECASE,
+ )
+ if not m:
+ return []
+ chunk = m.group(1)
+ chunk = re.sub(
+ r"^\s*Date\s+Number\s+Amount\s*$",
+ "",
+ chunk,
+ flags=re.MULTILINE | re.IGNORECASE,
+ )
+ _triplet = re.compile(
+ r"(\d{2}-\d{2})\s+(\*?\d+)\s+(\d{1,3}(?:,\d{3})*\.\d{2})"
+ )
+ out: List[Tuple[str, str, str]] = []
+ seen = set()
+ for t in _triplet.findall(chunk):
+ if t not in seen:
+ seen.add(t)
+ out.append(t)
+ return out
+
+def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
+ if not text_layer:
+ return ""
+
+ _DATE_RE = re.compile(r"^\d{2}-\d{2}$")
+ _MONEY_RE = re.compile(r"^\d{1,3}(?:,\d{3})*\.\d{2}$")
+ _CHECK_RE = re.compile(r"^\*?\d+$")
+
+ lines = [l.strip() for l in text_layer.splitlines()]
+ output = []
+
+ if needs_checks:
+ checks = []
+ i = 0
+ while i < len(lines):
+ if lines[i].lower().startswith("checks"):
+ i += 1
+ while i < len(lines):
+ if "daily account balance" in lines[i].lower():
+ break
+ if lines[i] == "Date":
+ i += 1
+ dates = []
+ while i < len(lines) and _DATE_RE.match(lines[i]):
+ dates.append(lines[i]); i += 1
+ while i < len(lines) and lines[i] != "Number":
+ i += 1
+ i += 1
+ numbers = []
+ while i < len(lines):
+ if _CHECK_RE.match(lines[i]):
+ numbers.append(lines[i]); i += 1
+ elif lines[i] == "":
+ i += 1
+ else:
+ break
+ while i < len(lines) and lines[i] != "Amount":
+ i += 1
+ i += 1
+ amounts = []
+ while i < len(lines):
+ if _MONEY_RE.match(lines[i]):
+ amounts.append(lines[i]); i += 1
+ elif lines[i] == "":
+ i += 1
+ else:
+ break
+ for d, n, a in zip(dates, numbers, amounts):
+ checks.append((d, n, a))
+ else:
+ i += 1
+ break
+ i += 1
+
+ if not checks and re.search(r"\bChecks\s*\(", text_layer, re.I):
+ checks = _jb_extract_checks_fallback(text_layer)
+
+ if checks:
+ rows = "\n".join(
+ " | | " + d + " | " + n + " | " + a + " |
"
+ for d, n, a in checks
+ )
+ output.append("Checks")
+ output.append(
+ '\n| Date | Number | Amount |
|---|
\n'
+ + rows + "\n
"
+ )
+
+ if needs_dab:
+ balances = []
+ i = 0
+ while i < len(lines):
+ if "daily account balance" in lines[i].lower():
+ i += 1
+ while i < len(lines):
+ if lines[i] == "Date":
+ i += 1
+ dates = []
+ while i < len(lines) and _DATE_RE.match(lines[i]):
+ dates.append(lines[i]); i += 1
+ while i < len(lines) and lines[i] != "Balance":
+ i += 1
+ i += 1
+ bals = []
+ while i < len(lines):
+ if _MONEY_RE.match(lines[i]):
+ bals.append(lines[i]); i += 1
+ elif lines[i] == "":
+ i += 1
+ else:
+ break
+ for d, b in zip(dates, bals):
+ balances.append((d, b))
+ else:
+ i += 1
+ break
+ i += 1
+
+ if balances:
+ rows = "\n".join(
+ "| " + d + " | " + b + " |
"
+ for d, b in balances
+ )
+ output.append("Daily Account Balance")
+ output.append(
+ "\n| Date | Balance |
|---|
\n"
+ + rows + "\n
"
+ )
+
+ return "\n\n".join(output)
+
+def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str:
+ if not page_md or "]*>.*?
", re.DOTALL | re.IGNORECASE)
+ result = page_md
+
+ for tbl_match in table_pattern.finditer(page_md):
+ table_html = tbl_match.group(0)
+
+ p = TableGridParser()
+ p.feed(table_html)
+ grid = _build_grid(p.rows)
+ if len(grid) < 2:
+ continue
+
+ header = [str(c or "").strip().upper() for c in grid[0]]
+ date_col = desc_col = amt_col = None
+ for ci, h in enumerate(header):
+ if re.search(r"\bDATE\b", h) and date_col is None:
+ date_col = ci
+ elif re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", h) and desc_col is None:
+ desc_col = ci
+ elif re.search(r"\b(AMOUNT|DEBIT|CREDIT|DR|CR)\b", h) and amt_col is None:
+ amt_col = ci
+
+ if date_col is None or desc_col is None or amt_col is None:
+ continue
+
+ ocr_freq = {}
+ ocr_rows_indexed = []
+ for ri, row in enumerate(grid[1:], start=1):
+ cells = [str(c or "").strip() for c in row]
+ if len(cells) <= max(date_col, desc_col, amt_col):
+ continue
+ d = _norm(cells[date_col])
+ desc = _norm(cells[desc_col])
+ amt = _norm(cells[amt_col])
+ if not d or not desc:
+ continue
+ key = (d, desc, amt)
+ ocr_freq[key] = ocr_freq.get(key, 0) + 1
+ ocr_rows_indexed.append((ri, key))
+
+ overlap = set(ocr_freq.keys()) & set(tl_freq.keys())
+
+ _ocr_dates = [
+ _norm(str(row[date_col] or ""))
+ for row in grid[1:]
+ if len(row) > date_col and str(row[date_col] or "").strip()
+ ]
+ _tl_dates = [_norm(r["date"]) for r in tl_rows]
+ _slash_re = re.compile(r"^\d{1,2}/\d{2}")
+ _hyphen_re = re.compile(r"^\d{1,2}-\d{2}")
+ def _date_fmt(dates):
+ if any(_slash_re.match(d) for d in dates): return "slash"
+ if any(_hyphen_re.match(d) for d in dates): return "hyphen"
+ return "other"
+ ocr_fmt = _date_fmt(_ocr_dates)
+ tl_fmt = _date_fmt(_tl_dates)
+ date_fmt_match = (ocr_fmt == tl_fmt) or "other" in (ocr_fmt, tl_fmt)
+
+ to_inject = {}
+ to_inject_new = {}
+
+ for key in tl_freq:
+ ocr_count = ocr_freq.get(key, 0)
+ tl_count = tl_freq[key]
+ if tl_count > ocr_count:
+ missing = tl_count - ocr_count
+ if ocr_count > 0:
+ if overlap:
+ to_inject[key] = missing
+ else:
+ if date_fmt_match:
+ tl_row_data = next(
+ (r for r in tl_rows
+ if (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"])) == key),
+ None
+ )
+ if tl_row_data:
+ to_inject_new[key] = (missing, tl_row_data)
+
+ if not to_inject and not to_inject_new:
+ continue
+
+ new_grid = [grid[0]]
+ for ri, row in enumerate(grid[1:], start=1):
+ new_grid.append(row)
+ cells = [str(c or "").strip() for c in row]
+ if len(cells) <= max(date_col, desc_col, amt_col):
+ continue
+ d = _norm(cells[date_col])
+ desc = _norm(cells[desc_col])
+ amt = _norm(cells[amt_col])
+ key = (d, desc, amt)
+ if key in to_inject and to_inject[key] > 0:
+ last_occ = max(idx for idx, k in ocr_rows_indexed if k == key)
+ if ri == last_occ:
+ for _ in range(to_inject[key]):
+ new_grid.append(list(row))
+ to_inject[key] = 0
+
+ new_table_html = _grid_to_html(new_grid)
+ result = result.replace(table_html, new_table_html, 1)
+
+ if to_inject_new:
+ ncols = len(grid[0])
+ header_row = grid[0]
+
+ post_date_col = None
+ for ci, h in enumerate(header_row):
+ hh = str(h or "").strip().upper()
+ if "POST" in hh and "DATE" in hh:
+ post_date_col = ci
+ break
+
+ new_rows = []
+ for key, (count, tl_row_data) in to_inject_new.items():
+ for _ in range(count):
+ new_row = [""] * ncols
+ new_row[date_col] = tl_row_data["date"]
+ new_row[desc_col] = tl_row_data["desc"]
+ new_row[amt_col] = tl_row_data["amount"]
+ if post_date_col is not None:
+ new_row[post_date_col] = tl_row_data.get("post_date", "")
+ new_rows.append(new_row)
+ log.info(
+ "page %d: new table row from text layer: %s %s",
+ page_num, tl_row_data["date"], tl_row_data["desc"][:40]
+ )
+
+ if new_rows:
+ extra_grid = [header_row] + new_rows
+ extra_html = "\n\n" + _grid_to_html(extra_grid)
+ insert_pos = result.find(new_table_html) + len(new_table_html)
+ result = result[:insert_pos] + extra_html + result[insert_pos:]
+
+ if tl_rows:
+ _has_txn_table = False
+ for _tbl in table_pattern.finditer(result):
+ _p2 = TableGridParser()
+ _p2.feed(_tbl.group(0))
+ _g2 = _build_grid(_p2.rows)
+ if len(_g2) < 2:
+ continue
+ _h2 = [str(c or "").strip().upper() for c in _g2[0]]
+ if (any(re.search(r"\bDATE\b", x) for x in _h2) and
+ any(re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", x) for x in _h2)):
+ _has_txn_table = True
+ break
+
+ if not _has_txn_table:
+ _has_post = any(r.get("post_date") for r in tl_rows)
+ if _has_post:
+ _chdr = ["Date", "Post Date", "Transaction Description", "Amount"]
+ _di2, _pi2, _xi2, _ai2 = 0, 1, 2, 3
+ else:
+ _chdr = ["Date", "Transaction Description", "Amount"]
+ _di2, _xi2, _ai2 = 0, 1, 2
+ _cnew_rows = []
+ for r in tl_rows:
+ _crow = [""] * len(_chdr)
+ _crow[_di2] = r["date"]
+ _crow[_xi2] = r["desc"]
+ _crow[_ai2] = r["amount"]
+ if _has_post:
+ _crow[_pi2] = r.get("post_date", "")
+ _cnew_rows.append(_crow)
+ result = _grid_to_html([_chdr] + _cnew_rows) + "\n\n" + result
+ log.info("page %d: Case C — new table (%d rows) from text layer",
+ page_num, len(_cnew_rows))
+
+ return result
+
+ except Exception as e:
+ log.warning("_patch_ocr_with_textlayer failed (page %d): %s", page_num, e)
+ _tl_freq_context.clear()
+ return page_md
+
+# --------------------------
+# Fix 5: Mid-table separator row reconstruction
+# --------------------------
+
+_DATE_RE = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
+_SPLIT_AMOUNT_RE = re.compile(
+ r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
+)
+
+def _reconstruct_separator_rows(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ ncols = len(grid[0])
+ if ncols < 2:
+ return grid
+
+ new_grid = [grid[0]]
+
+ for row in grid[1:]:
+ cells = [str(c or "").strip() for c in row]
+ while len(cells) < ncols:
+ cells.append("")
+
+ non_empty = [(i, c) for i, c in enumerate(cells) if c]
+
+ if len(non_empty) == 0:
+ new_grid.append(row)
+ continue
+
+ first_idx, first_val = non_empty[0]
+
+ if _DATE_RE.match(first_val):
+ new_grid.append(row)
+ continue
+
+ if not re.search(r"[A-Za-z]", first_val):
+ new_grid.append(row)
+ continue
+
+ if len(non_empty) == 1:
+ new_row = [""] * ncols
+ new_row[0] = first_val
+ new_grid.append(new_row)
+ continue
+
+ trailing = [val for _, val in non_empty[1:]]
+ all_trailing_are_digit_fragments = all(
+ re.fullmatch(r"\d{1,4}", v) for v in trailing
+ )
+
+ if not all_trailing_are_digit_fragments:
+ new_grid.append(row)
+ continue
+
+ reconstructed = "".join(val for _, val in non_empty).strip()
+ new_row = [""] * ncols
+ new_row[0] = reconstructed
+ new_grid.append(new_row)
+
+ return new_grid
+
+# --------------------------
+# Normalizer entry point
+# --------------------------
+
+def _merge_split_rows(grid):
+ if not grid or len(grid) < 3:
+ return grid
+
+ ncols = len(grid[0])
+ if ncols < 3:
+ return grid
+
+ header = [str(c or "").strip().upper() for c in grid[0]]
+ date_col = desc_col = bal_col = None
+ debit_cols = []
+ credit_cols = []
+
+ for ci, h in enumerate(header):
+ if re.search(r"\bDATE\b", h) and date_col is None:
+ date_col = ci
+ if re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?|TRANSACTION)\b", h) and desc_col is None:
+ desc_col = ci
+ if re.search(r"\bBALANCE\b", h) and bal_col is None:
+ bal_col = ci
+ if re.search(r"\b(DEBIT|DEBITS|DR|WITHDRAWAL|SUBTRACTIONS?)\b", h):
+ debit_cols.append(ci)
+ if re.search(r"\b(CREDIT|CREDITS|CR|DEPOSIT|ADDITIONS?)\b", h):
+ credit_cols.append(ci)
+
+ if date_col is None or desc_col is None:
+ return grid
+ if not debit_cols and not credit_cols:
+ return grid
+
+ money_cols = debit_cols + credit_cols
+ _MONEY_RE2 = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*\.\d{1,4}$")
+ _DATE_RE2 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$|^\d{1,2}-\d{2}(?:-\d{2,4})?$")
+ _FRAGMENT_RE = re.compile(r"^[^\s\.]{1,6}$")
+
+ new_grid = [grid[0]]
+ i = 1
+ while i < len(grid):
+ row = [str(c or "").strip() for c in grid[i]]
+ while len(row) < ncols:
+ row.append("")
+
+ has_desc = bool(row[desc_col])
+ no_date = not row[date_col]
+ no_money = all(not row[c] for c in money_cols)
+ bal_is_frag = (bal_col is not None and
+ (_FRAGMENT_RE.match(row[bal_col]) or not row[bal_col]))
+
+ if has_desc and no_date and no_money and bal_is_frag and (i + 1) < len(grid):
+ next_row = [str(c or "").strip() for c in grid[i + 1]]
+ while len(next_row) < ncols:
+ next_row.append("")
+
+ next_has_date = bool(next_row[date_col]) and _DATE_RE2.match(next_row[date_col])
+ next_has_money = any(bool(next_row[c]) for c in money_cols)
+
+ if next_has_date and next_has_money:
+ merged = list(next_row)
+ if next_row[desc_col]:
+ merged[desc_col] = row[desc_col] + " " + next_row[desc_col]
+ else:
+ merged[desc_col] = row[desc_col]
+ new_grid.append(merged)
+ i += 2
+ continue
+
+ new_grid.append(row)
+ i += 1
+
+ return new_grid
+
+def _extract_fused_desc_amount(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ ncols = len(grid[0])
+ header = [str(c or "").strip().upper() for c in grid[0]]
+
+ date_col = desc_col = None
+ debit_cols = []
+ credit_cols = []
+ for ci, h in enumerate(header):
+ if re.search(r"\bDATE\b", h) and date_col is None:
+ date_col = ci
+ if re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?|TRANSACTION)\b", h) and desc_col is None:
+ desc_col = ci
+ if re.search(r"\b(DEBIT|DEBITS|DR|WITHDRAWAL|SUBTRACTIONS?)\b", h):
+ debit_cols.append(ci)
+ if re.search(r"\b(CREDIT|CREDITS|CR|DEPOSIT|ADDITIONS?)\b", h):
+ credit_cols.append(ci)
+
+ if date_col is None or desc_col is None or not debit_cols:
+ return grid
+
+ money_cols = debit_cols + credit_cols
+ _TRAILING_AMT = re.compile(r"^(.+?)\s+(-?\d{1,3}(?:,\d{3})*\.\d{2})$")
+ debit_col = debit_cols[0]
+
+ new_grid = [grid[0]]
+ for row in grid[1:]:
+ cells = [str(c or "").strip() for c in row]
+ while len(cells) < ncols:
+ cells.append("")
+
+ desc = cells[desc_col]
+ no_money = all(not cells[c] for c in money_cols)
+ has_date = bool(cells[date_col])
+
+ if desc and no_money and has_date:
+ m = _TRAILING_AMT.match(desc)
+ if m:
+ cells = list(cells)
+ cells[desc_col] = m.group(1).strip()
+ cells[debit_col] = m.group(2)
+ new_grid.append(cells)
+ continue
+
+ new_grid.append(cells)
+
+ return new_grid
+
+def _is_junk_table(grid):
+ if not grid or len(grid) < 2:
+ return False
+ ncols = len(grid[0])
+ if ncols != 1:
+ return False
+ header_text = str(grid[0][0] or "").strip().upper()
+ if "CUSTOMER INFORMATION" in header_text:
+ data_rows = grid[1:]
+ if all(len(str(r[0] or "").strip()) <= 10 for r in data_rows):
+ return True
+ return False
+
+def _split_fused_multicolumn_header(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ ncols = len(grid[0])
+ if ncols < 2:
+ return grid
+
+ first_header = str(grid[0][0] or "").strip()
+ parts = first_header.split()
+ kw_hits = [p for p in parts if _HEADER_KW_EXACT_RE.match(p)]
+ if len(kw_hits) < 2:
+ return grid
+
+ rest_headers = [str(c or "").strip() for c in grid[0][1:]]
+ if not all(_HEADER_KW_EXACT_RE.match(h) for h in rest_headers if h):
+ return grid
+
+ new_header_cols = []
+ non_kw_parts = []
+ for p in parts:
+ if _HEADER_KW_EXACT_RE.match(p):
+ if non_kw_parts:
+ new_header_cols.append(" ".join(non_kw_parts))
+ non_kw_parts = []
+ new_header_cols.append(p)
+ else:
+ non_kw_parts.append(p)
+ if non_kw_parts:
+ new_header_cols.append(" ".join(non_kw_parts))
+
+ new_header = new_header_cols + rest_headers
+ new_ncols = len(new_header)
+ extra_cols = new_ncols - ncols
+
+ _DATE_COL_RE = re.compile(r"\bDATE\b", re.IGNORECASE)
+ _CARD_COL_RE = re.compile(r"\bCARD\b", re.IGNORECASE)
+ _DESC_COL_RE = re.compile(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", re.IGNORECASE)
+ _MONEY_COL_RE = re.compile(r"\b(DEPOSIT|WITHDRAWAL|DEBIT|CREDIT|AMOUNT|ADDITION|SUBTRACTION)S?\b", re.IGNORECASE)
+
+ new_date_col = next((i for i,h in enumerate(new_header) if _DATE_COL_RE.search(h)), None)
+ new_desc_col = next((i for i,h in enumerate(new_header) if _DESC_COL_RE.search(h)), None)
+ new_card_col = next((i for i,h in enumerate(new_header) if _CARD_COL_RE.search(h)), None)
+
+ if new_date_col is None or new_desc_col is None:
+ return grid
+
+ _DATE_PREFIX = re.compile(r"^(\d{1,2}/\d{2}(?:/\d{2,4})?)\s+(.*)", re.DOTALL)
+ _CARD_SUFFIX = re.compile(r"^(.*?)\s+(\d{4})$", re.DOTALL)
+
+ def pad(row, n):
+ r = list(row)
+ while len(r) < n: r.append("")
+ return r[:n]
+
+ new_grid = [pad(new_header, new_ncols)]
+ for row in grid[1:]:
+ cells = [str(c or "").strip() for c in row]
+ while len(cells) < ncols: cells.append("")
+
+ fused_cell = cells[0]
+ rest_cells = cells[1:]
+
+ date_val = ""
+ desc_val = fused_cell
+ card_val = ""
+
+ dm = _DATE_PREFIX.match(fused_cell)
+ if dm:
+ date_val = dm.group(1)
+ desc_val = dm.group(2).strip()
+
+ if new_card_col is not None:
+ cm = _CARD_SUFFIX.match(desc_val)
+ if cm:
+ desc_val = cm.group(1).strip()
+ card_val = cm.group(2)
+
+ new_row = [""] * new_ncols
+ new_row[new_date_col] = date_val
+ new_row[new_desc_col] = desc_val
+ if new_card_col is not None:
+ new_row[new_card_col] = card_val
+ money_col_indices = [i for i,h in enumerate(new_header) if _MONEY_COL_RE.search(h)]
+ for mi, ri in enumerate(range(len(rest_cells))):
+ if mi < len(money_col_indices):
+ new_row[money_col_indices[mi]] = rest_cells[ri]
+
+ new_grid.append(new_row)
+
+ return new_grid
+
+def _normalize_daily_balance_table(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ header = [str(c or "").strip().upper() for c in grid[0]]
+ ncols = len(header)
+
+ if ncols < 4 or ncols % 2 != 0:
+ return grid
+
+ half = ncols // 2
+ date_half = header[:half]
+ bal_half = header[half:]
+
+ if not all(h == "DATE" for h in date_half):
+ return grid
+ if not all(h == "BALANCE" for h in bal_half):
+ return grid
+
+ new_header = []
+ for i in range(half):
+ new_header.append("DATE")
+ new_header.append("BALANCE")
+
+ new_rows = []
+ _DATE_RE3 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
+
+ for row in grid[1:]:
+ cells = [str(c or "").strip() for c in row]
+ while len(cells) < ncols:
+ cells.append("")
+
+ date_cells = [cells[0]]
+ bal_cells = cells[1:]
+
+ first = date_cells[0]
+ tokens = re.split(r"[\n\r\s]+", first)
+ date_tokens = [t.strip() for t in tokens if t.strip() and _DATE_RE3.match(t.strip())]
+ if len(date_tokens) >= 2:
+ stacked = date_tokens
+ else:
+ stacked = date_cells
+
+ new_row = [""] * ncols
+ for i, date_val in enumerate(stacked):
+ if not _DATE_RE3.match(date_val):
+ continue
+ bal_val = bal_cells[i] if i < len(bal_cells) else ""
+ new_row[i * 2] = date_val
+ new_row[i * 2 + 1] = bal_val
+ new_rows.append(new_row)
+
+ if not new_rows:
+ return grid
+
+ return [new_header] + new_rows
+
+def _normalize_multiline_register_header(grid):
+ """
+ Normalize two-row register headers where row0 is mostly blank/Amount placeholders and
+ row1 carries the real labels (DATE, DESCRIPTION, DEPOSIT, WITHDRAWAL, CARD #).
+ """
+ if not grid or len(grid) < 3:
+ return grid
+ r0 = [str(c or "").strip() for c in grid[0]]
+ r1 = [str(c or "").strip() for c in grid[1]]
+ if not r1:
+ return grid
+
+ r1_join = " ".join(r1).upper()
+ if "DATE" not in r1_join:
+ return grid
+ if "WITHDRAWAL" not in r1_join and "DEPOSIT" not in r1_join:
+ return grid
+ if "DESCRIPTION" not in r1_join:
+ return grid
+
+ # Guard: only treat as multiline header if row0 is mostly empty / generic title-ish cells.
+ non_empty_r0 = [x for x in r0 if x]
+ if len(non_empty_r0) > max(3, len(r0) // 2):
+ if not ("ACCOUNT HISTORY" in " ".join(r0).upper()):
+ return grid
+ generic_r0 = all(
+ (not x) or (x.upper() in ("AMOUNT", "ACCOUNT HISTORY", "DATE", "DESCRIPTION", "DEPOSIT", "WITHDRAWAL", "CARD #"))
+ for x in r0
+ )
+ if not generic_r0:
+ return grid
+
+ # Use row1 as canonical header, keeping column count stable.
+ ncols = max(len(r0), len(r1))
+ new_header = []
+ for i in range(ncols):
+ v = (r1[i] if i < len(r1) else "").strip()
+ if not v and i < len(r0):
+ v = str(r0[i] or "").strip()
+ new_header.append(v)
+
+ out = [new_header]
+ out.extend(grid[2:])
+ return out
+
+def _normalize_checks_paid_table(grid):
+ if not grid or len(grid) < 2:
+ return grid
+
+ ncols = len(grid[0])
+ if ncols < 2:
+ return grid
+
+ first_h = str(grid[0][0] or "").strip()
+ _CHECKS_HDR = re.compile(r"^(DATE\s+CHECK\s+#\s*)+$", re.IGNORECASE)
+ if not _CHECKS_HDR.match(first_h):
+ return grid
+
+ rest_h = [str(c or "").strip().upper() for c in grid[0][1:]]
+ if not all(h in ("AMOUNT", "") for h in rest_h if h):
+ return grid
+
+ n_groups = len(re.findall(r"\bDATE\b", first_h, re.IGNORECASE))
+ if n_groups < 1:
+ return grid
+
+ new_header = []
+ for _ in range(n_groups):
+ new_header.extend(["DATE", "CHECK #", "AMOUNT"])
+
+ _DATE_RE = re.compile(r"^\d{1,2}/\d{2}$")
+
+ new_rows = [new_header]
+ for row in grid[1:]:
+ cells = [str(c or "").strip() for c in row]
+ while len(cells) < ncols:
+ cells.append("")
+
+ fused_data = cells[0]
+ amounts = cells[1:]
+
+ tokens = fused_data.split()
+ groups = []
+ current = []
+ for tok in tokens:
+ if _DATE_RE.match(tok) and current:
+ groups.append(current)
+ current = [tok]
+ else:
+ current.append(tok)
+ if current and current[0]:
+ groups.append(current)
+
+ new_row = [""] * len(new_header)
+ for i, grp in enumerate(groups):
+ if i >= n_groups:
+ break
+ new_row[i * 3] = grp[0]
+ new_row[i * 3 + 1] = " ".join(grp[1:]).strip()
+ new_row[i * 3 + 2] = amounts[i] if i < len(amounts) else ""
+
+ new_rows.append(new_row)
+
+ return new_rows
+
+_JB_SECTION_HDR = re.compile(
+ r"^(Deposits?|Withdrawals?|Checks?(?:\s+Paid)?|Daily\s+Account\s+Balance)"
+ r"\s*(?:\(cont\.?\))?\s*$",
+ re.IGNORECASE,
+)
+_JB_COL_HDR = re.compile(
+ r"^Date\s+(Description|Number)\s+Amount\s*$"
+ r"|^Date\s+(?:Number\s+)?(?:Balance|Amount)\s*$",
+ re.IGNORECASE,
+)
+_JB_DATE_DD = re.compile(r"^\d{2}-\d{2}\s+")
+_JB_TXN_ROW = re.compile(r"^(\d{2}-\d{2})\s+(.+?)\s+(-?\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
+_JB_BAL_ROW = re.compile(r"^(\d{2}-\d{2})\s+(\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
+
+def _jb_rows_to_html(header_cols, rows):
+ th = "".join("
" + h + " | " for h in header_cols)
+ parts = ["
", "" + th + "
"]
+ for row in rows:
+ td = "".join("" + str(row.get(h, "")) + " | " for h in header_cols)
+ parts.append("" + td + "
")
+ parts.append("
")
+ return "\n".join(parts)
+
+def _jb_section_has_table(lines, start, end):
+ return any("
str:
+ lines = text.splitlines()
+ out = []
+ i = 0
+
+ while i < len(lines):
+ line = lines[i].strip()
+
+ if not _JB_SECTION_HDR.match(line):
+ out.append(lines[i])
+ i += 1
+ continue
+
+ section_title = line
+ j = i + 1
+
+ while j < len(lines) and not lines[j].strip():
+ j += 1
+
+ if j >= len(lines) or not _JB_COL_HDR.match(lines[j].strip()):
+ out.append(lines[i])
+ i += 1
+ continue
+
+ col_hdr_line = lines[j].strip()
+ data_start = j + 1
+
+ first_data_idx = _find_first_data_row(lines, data_start)
+
+ if first_data_idx is None or not _JB_DATE_DD.match(lines[first_data_idx].strip()):
+ out.append(lines[i])
+ i += 1
+ continue
+
+ section_end = len(lines)
+ for k in range(data_start, len(lines)):
+ l = lines[k].strip()
+ if (l != section_title and _JB_SECTION_HDR.match(l)
+ or l.startswith("---page-separator")
+ or l.startswith("Page:")):
+ section_end = k
+ break
+
+ if _jb_section_has_table(lines, i, section_end):
+ out.append(lines[i])
+ i += 1
+ continue
+
+ parts_upper = col_hdr_line.upper().split()
+ if len(parts_upper) == 2 and parts_upper[1] == "BALANCE":
+ header_cols = ["Date", "Balance"]
+ elif "NUMBER" in parts_upper:
+ header_cols = ["Date", "Number", "Amount"]
+ else:
+ header_cols = ["Date", "Description", "Amount"]
+
+ out.append(section_title)
+ rows = []
+ current = None
+ k = data_start
+
+ while k < section_end:
+ raw = lines[k]
+ row_line = raw.strip()
+
+ if not row_line:
+ k += 1
+ continue
+
+ if (_JB_SECTION_HDR.match(row_line)
+ or row_line.startswith("---page-separator")
+ or row_line.startswith("Page:")):
+ break
+
+ mb = _JB_BAL_ROW.match(row_line)
+ if mb and header_cols == ["Date", "Balance"]:
+ if current:
+ rows.append(current)
+ current = {"Date": mb.group(1), "Balance": mb.group(2)}
+ k += 1
+ continue
+
+ mt = _JB_TXN_ROW.match(row_line)
+ if mt:
+ if current:
+ rows.append(current)
+ desc_key = "Description" if "Description" in header_cols else "Number"
+ current = {
+ "Date": mt.group(1),
+ desc_key: mt.group(2).strip(),
+ "Amount": mt.group(3),
+ }
+ k += 1
+ continue
+
+ if current and row_line:
+ desc_key = "Description" if "Description" in header_cols else "Number"
+ if desc_key in current:
+ current[desc_key] += " " + row_line
+ k += 1
+ continue
+
+ k += 1
+
+ if current:
+ rows.append(current)
+
+ if rows:
+ out.append(_jb_rows_to_html(header_cols, rows))
+
+ i = section_end
+ continue
+
+ return "\n".join(out)
+
+# --------------------------
+# Navy Federal full-page text-layer parser (Fix NF-1)
+# --------------------------
+
+def _is_nfcu_document(pdf_path: str) -> bool:
+ try:
+ import pymupdf as fitz
+ doc = fitz.open(pdf_path)
+ pages_to_check = min(2, len(doc))
+ text = ""
+ for i in range(pages_to_check):
+ text += "\n" + (doc[i].get_text() or "")
+ doc.close()
+ t = text.lower()
+ return (
+ ("statement of account" in t)
+ and ("access no." in t)
+ and ("routing number" in t)
+ and ("navy federal" in t or "navyfederal.org" in t)
+ )
+ except Exception:
+ return False
+
+
+def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
+ try:
+ import pymupdf as fitz
+
+ doc = fitz.open(pdf_path)
+ if page_num >= len(doc):
+ doc.close()
+ return ""
+ page = doc[page_num]
+ page_text = page.get_text() or ""
+ words = page.get_text("words") or []
+ doc.close()
+
+ if not page_text or not words:
+ return ""
+
+ t = page_text.lower()
+ is_nfcu = (
+ ("statement of account" in t)
+ and (("access no." in t) or ("routing number" in t) or ("navy federal" in t) or ("navyfederal.org" in t))
+ )
+ if not is_nfcu:
+ return ""
+
+ buckets = []
+ for w in words:
+ if len(w) < 5:
+ continue
+ x0 = float(w[0])
+ y0 = float(w[1])
+ token = w[4].strip()
+ if not token:
+ continue
+ bi = None
+ for i, (yb, _) in enumerate(buckets):
+ if abs(yb - y0) <= 2.0:
+ bi = i
+ break
+ if bi is None:
+ buckets.append((y0, [(x0, token)]))
+ else:
+ buckets[bi][1].append((x0, token))
+
+ lines = []
+ for y, arr in sorted(buckets, key=lambda x: x[0]):
+ toks = [tok for _, tok in sorted(arr, key=lambda x: x[0])]
+ xs = [x for x, _ in sorted(arr, key=lambda x: x[0])]
+ lines.append((y, xs, toks, " ".join(toks)))
+
+ date_re = re.compile(r"^\d{2}-\d{2}$")
+ money_re = re.compile(r"^[\d,]+\.\d{2}-?$")
+ acct_re = re.compile(r"^(Business Checking|Mbr Business Savings)\s*-\s*(\d+)")
+ stop_markers = (
+ "Items Paid",
+ "Average Daily Balance",
+ "2024 Year to Date",
+ "Disclosure",
+ "What to Do",
+ "Errors",
+ "Payments",
+ "SAVINGS DIVIDENDS",
+ "REMITTANCE RECEIVED",
+ "DEPOSIT VOUCHER",
+ "ACCOUNT NUMBER",
+ "MARK \"X\"",
+ "ADDRESS/ORDER",
+ "ITEMS ON REVERSE",
+ "CHANGE OF ADDRESS",
+ "PLEASE PRINT",
+ "RANK/RATE",
+ "PO BOX",
+ "MERRIFIELD",
+ "Questions about this Statement",
+ )
+
+ def parse_account_rows(start_idx):
+ rows = []
+ i = start_idx
+ pending = None
+ while i < len(lines):
+ _, _, toks, text = lines[i]
+ if not text:
+ i += 1
+ continue
+
+ if acct_re.match(text) and i != start_idx:
+ break
+ if any(text.startswith(s) for s in stop_markers):
+ break
+ upper_text = text.upper()
+ if (
+ "REMITTANCE RECEIVED" in upper_text
+ or "DEPOSIT VOUCHER" in upper_text
+ or "ACCOUNT NUMBER" in upper_text
+ or "ADDRESS/ORDER" in upper_text
+ or "ITEMS ON REVERSE" in upper_text
+ or "CHANGE OF ADDRESS" in upper_text
+ or "PLEASE PRINT" in upper_text
+ ):
+ break
+ if text in ("Checking", "Savings"):
+ break
+ if text.startswith("Date Transaction Detail"):
+ i += 1
+ continue
+
+ def flush_pending():
+ nonlocal pending
+ if pending is not None:
+ rows.append(pending)
+ pending = None
+
+ if toks and date_re.match(toks[0]):
+ flush_pending()
+ date = toks[0]
+ rest = toks[1:]
+ mvals = [tok for tok in rest if money_re.match(tok)]
+ desc_tokens = [tok for tok in rest if not money_re.match(tok)]
+
+ amount = ""
+ balance = ""
+ if len(mvals) >= 2:
+ amount = mvals[-2]
+ balance = mvals[-1]
+ elif len(mvals) == 1:
+ balance = mvals[-1]
+
+ pending = {
+ "date": date,
+ "desc": " ".join(desc_tokens).strip(),
+ "amount": amount,
+ "balance": balance,
+ }
+ else:
+ if pending is not None:
+ mvals = [tok for tok in toks if money_re.match(tok)]
+ desc_tokens = [tok for tok in toks if not money_re.match(tok)]
+
+ upper = text.upper()
+ if pending["balance"] and (
+ "REMITTANCE" in upper
+ or "DEPOSIT VOUCHER" in upper
+ or "ACCOUNT NUMBER" in upper
+ or "ADDRESS/ORDER" in upper
+ or "ITEMS ON REVERSE" in upper
+ or "PO BOX" in upper
+ ):
+ break
+
+ if (
+ ("BEGINNING BALANCE" in pending["desc"].upper() or "ENDING BALANCE" in pending["desc"].upper())
+ and not mvals
+ and not (toks and date_re.match(toks[0]))
+ ):
+ break
+ if desc_tokens:
+ pending["desc"] = (pending["desc"] + " " + " ".join(desc_tokens)).strip()
+ if len(mvals) >= 2 and (not pending["amount"] or not pending["balance"]):
+ if not pending["amount"]:
+ pending["amount"] = mvals[-2]
+ if not pending["balance"]:
+ pending["balance"] = mvals[-1]
+ elif len(mvals) == 1 and not pending["balance"]:
+ pending["balance"] = mvals[-1]
+ i += 1
+
+ if pending is not None:
+ rows.append(pending)
+ cleaned = [r for r in rows if r["date"] and r["desc"] and (r["amount"] or r["balance"])]
+ return cleaned, i
+
+ out = []
+ i = 0
+ while i < len(lines):
+ _, _, _, text = lines[i]
+ if not text:
+ i += 1
+ continue
+
+ m = acct_re.match(text)
+ if m:
+ acct_name = m.group(1)
+ acct_no = m.group(2)
+ title = f"{acct_name} - {acct_no}"
+ if i + 1 < len(lines) and "Continued" in lines[i + 1][3]:
+ title += " (Continued from previous page)"
+ out.append(f"{title}")
+ out.append("")
+ out.append("| Date | Transaction Detail | Amount($) | Balance($) |
|---|
")
+ rows, i2 = parse_account_rows(i + 1)
+ for r in rows:
+ out.append(
+ f"| {r['date']} | {r['desc']} | {r['amount']} | {r['balance']} |
"
+ )
+ out.append("
")
+ i = i2
+ continue
+
+ if text in ("Checking", "Savings"):
+ out.append(f"{text}
")
+ elif text.startswith("Items Paid"):
+ out.append("Items Paid")
+ i += 1
+
+ return "\n".join(out).strip()
+
+ except Exception as e:
+ log.warning("_parse_nfcu_page failed (page %d): %s", page_num, e)
+ return ""
+
+def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
+ try:
+ import pymupdf as fitz
+
+ doc = fitz.open(pdf_path)
+ if page_num >= len(doc):
+ doc.close()
+ return ""
+ page = doc[page_num]
+ words = page.get_text("words") or []
+ doc.close()
+ if not words:
+ return ""
+
+ buckets = []
+ for w in words:
+ if len(w) < 5:
+ continue
+ x0 = float(w[0])
+ y0 = float(w[1])
+ tok = str(w[4] or "").strip()
+ if not tok:
+ continue
+ bi = None
+ for i, (yb, _) in enumerate(buckets):
+ if abs(yb - y0) <= 2.0:
+ bi = i
+ break
+ if bi is None:
+ buckets.append((y0, [(x0, tok)]))
+ else:
+ buckets[bi][1].append((x0, tok))
+
+ lines = []
+ for y, arr in sorted(buckets, key=lambda x: x[0]):
+ arr2 = sorted(arr, key=lambda x: x[0])
+ toks = [t for _, t in arr2]
+ text = " ".join(toks).strip()
+ lines.append((y, toks, text))
+
+ anchor_idx = None
+ for i, (_, _, text) in enumerate(lines):
+ tl = text.lower()
+ if "summary of your deposit accounts" in tl or ("summary" in tl and "deposit accounts" in tl):
+ anchor_idx = i
+ break
+ if anchor_idx is None:
+ return ""
+
+ money_re = re.compile(r"^\$?[\d,]+\.\d{2}-?$")
+ acct_no_re = re.compile(r"^\d{8,12}$")
+ stop_re = re.compile(
+ r"(business checking\s*-|mbr business savings\s*-|date\s+transaction\s+detail|items paid)",
+ re.IGNORECASE,
+ )
+
+ rows = []
+ pending_account = ""
+ account_name_re = re.compile(
+ r"^(business checking|mbr business savings|checking|savings|money market|certificate).*$",
+ re.IGNORECASE,
+ )
+
+ for _, toks, text in lines[anchor_idx + 1:]:
+ if not text:
+ continue
+ if stop_re.search(text):
+ break
+ if not toks:
+ continue
+ t0 = toks[0].lower()
+ tl = text.lower()
+
+ if (
+ "previous" in tl
+ or "deposits" in tl
+ or "withdrawals" in tl
+ or "ending" in tl
+ or "dividends" in tl
+ ) and not any(money_re.match(t) for t in toks):
+ continue
+
+ if account_name_re.match(text.strip()) and not any(money_re.match(t) for t in toks):
+ pending_account = text.strip()
+ continue
+
+ mvals = [t.replace("$", "") for t in toks if money_re.match(t)]
+ acct_tokens = [t for t in toks if acct_no_re.match(t)]
+
+ if t0 == "totals" and len(mvals) >= 5:
+ rows.append(["Totals", mvals[0], mvals[1], mvals[2], mvals[3], mvals[4]])
+ continue
+
+ if acct_tokens and len(mvals) >= 5:
+ acct = acct_tokens[0]
+ acct_label = (pending_account + " - " + acct).strip(" -") if pending_account else acct
+ rows.append([acct_label, mvals[0], mvals[1], mvals[2], mvals[3], mvals[4]])
+ pending_account = ""
+ continue
+
+ if not rows:
+ return ""
+
+ hdr = [
+ "Account",
+ "Previous Balance",
+ "Deposits/Credits",
+ "Withdrawals/Debits",
+ "Ending Balance",
+ "YTD Dividends",
+ ]
+ return _grid_to_html([hdr] + rows)
+ except Exception:
+ return ""
+
+def _replace_nfcu_summary_table(page_md: str, summary_table_html: str) -> str:
+ if not page_md or not summary_table_html:
+ return page_md
+ table_pattern = re.compile(r"]*>.*?