app.py

#!/usr/bin/env python3
"""
Simplified GLM-OCR Hugging Face / local Gradio app.

Scope (intentionally small):
  - PDF → padded high-DPI page images → GLM-OCR body markdown
  - Header band: PDF text extraction first, optional header OCR fallback
  - Footer band: same pattern, with light dedup so we do not paste a full
    transaction dump twice when the body already captured it

Universal image pipeline (same for every PDF, no keywords / no bank logic):
  - Higher rasterization scale + extra white padding so fine print, boxed
    section labels, and right-aligned amounts sit farther from the clip edge.
  - Mild contrast + unsharp mask on every raster sent to the model so
    thin rules and small glyphs are easier to read before recognition.

Explicitly omitted vs the heavy Space build:
  - No text-layer row injection, institution-specific splits (UCB / Navy /
    TD / First Horizon / …), or doc-wide dedupe passes.

Included (data-driven, no institution names):
  - HTML tables: modal logical width from rowspan-free rows (colspan-aware);
    pad short rows; trim trailing empty cells; expand each row to a logical
    grid, then slide a solitary amount token past trailing blank logical slots
    into the rightmost slot (colspan-aware). Rowspan rows are skipped for edits
    but do not disable an entire table. Stabilization runs in multiple passes.
  - Split single cells that clearly contain transaction amount + trailing balance
    (two money tokens, tight gap) into two cells so classifiers can see a balance column.
  - thead uses th only; degenerate empty/sparse non-financial tables are dropped.

Configure GLMOCR_API_KEY, GLM_OCR_API_KEY, or ZHIPU_API_KEY (environment). Optional: glmocr + gradio +
pymupdf + pillow installed.
"""

# Patch asyncio first (before Gradio imports it) to reduce Python 3.13 loop noise
import asyncio

try:
    _orig_close = asyncio.BaseEventLoop.close

    def _safe_close(self):
        try:
            _orig_close(self)
        except (ValueError, OSError):
            pass

    asyncio.BaseEventLoop.close = _safe_close
except Exception:
    pass

import html
import logging
import os
import re
import tempfile
import uuid
from collections import Counter
from typing import List, Optional, Tuple

import yaml

try:
    import glmocr

    GLMOCR_BASE = os.path.dirname(glmocr.__file__)
    CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
except ImportError:
    glmocr = None  # type: ignore
    GLMOCR_BASE = ""
    CONFIG_PATH = ""

log = logging.getLogger("glmocr_simple_app")
logging.basicConfig(level=logging.INFO)

# ---------------------------------------------------------------------------
# Settings — tuned for dense financial PDFs; applies to every document
# ---------------------------------------------------------------------------

# Never commit secrets: Space / local runs use ZHIPU_API_KEY or GLMOCR_API_KEY.
GLMOCR_API_KEY = "cee1d52dd91a4ab591b3f6e105f8ad89.LgbQTECuzX0zrito"
if not GLMOCR_API_KEY:
    log.warning(
        "No ZHIPU_API_KEY or GLMOCR_API_KEY in environment; GlmOcr() will fail until you set one."
    )

# Rasterization: higher scale = more pixels per PDF point (helps small type,
# boxed headers, and narrow columns). Same constant for all uploads.
RENDER_SCALE = 3.0

# White margin as a fraction of page width/height after render. Extra right
# margin helps right-aligned currency columns that hug the page edge.
PAD_LEFT_FRAC = 0.035
PAD_RIGHT_FRAC = 0.10
PAD_TOP_FRAC = 0.018
PAD_BOTTOM_FRAC = 0.018

# Synthesized running-balance columns help some heuristics but downstream LLM
# extractors may mis-read them as credits; default off (set GLMOCR_SYNTH_RUNNING_BALANCE=1 to enable).
def _synth_running_balance_enabled() -> bool:
    return os.environ.get("GLMOCR_SYNTH_RUNNING_BALANCE", "").lower() in ("1", "true", "yes")


ENABLE_CONTRAST = True
# Slight contrast lift only; same factor for every file.
CONTRAST_FACTOR = 1.18

# Subtle edge enhancement after contrast (helps hairlines and small digits).
ENABLE_UNSHARP = True
UNSHARP_RADIUS = 0.78
UNSHARP_PERCENT = 76
UNSHARP_THRESHOLD = 1

DEFAULT_ZONE_FRAC = 0.12
PDF_HEADER_BAND_FRAC = 0.10

ENABLE_FOOTER_OCR = True
PDF_FOOTER_BAND_FRAC = 0.88

MIN_CROP_HEIGHT = 112
MIN_CROP_PIXELS = 112 * 112

# PNG compression 0–9; lower = less loss before GLM-OCR (same for all PDFs).
PAGE_PNG_COMPRESS_LEVEL = 3
# JPEG quality for small header/footer crops sent to the API.
ZONE_JPEG_QUALITY = 95

MIN_PDF_TEXT_CHARS_NATIVE_LAYER = 1500

_parser = None


def _enhance_raster_for_ocr(img):
    """
    Improve legibility of every raster passed to GLM-OCR (full pages and
    header/footer crops). No document text or keywords — same pipeline for
    all PDFs and images.
    """
    from PIL import ImageEnhance, ImageFilter

    if ENABLE_CONTRAST:
        img = ImageEnhance.Contrast(img).enhance(CONTRAST_FACTOR)
    if ENABLE_UNSHARP:
        img = img.filter(
            ImageFilter.UnsharpMask(
                radius=UNSHARP_RADIUS,
                percent=UNSHARP_PERCENT,
                threshold=UNSHARP_THRESHOLD,
            )
        )
    return img


def get_parser():
    global _parser
    if glmocr is None:
        raise RuntimeError("glmocr is not installed.")
    if _parser is None:
        from glmocr import GlmOcr

        kw = {"mode": "maas"}
        if GLMOCR_API_KEY:
            kw["api_key"] = GLMOCR_API_KEY
        _parser = GlmOcr(**kw)
    return _parser


if CONFIG_PATH:
    try:
        with open(CONFIG_PATH, "r", encoding="utf-8") as f:
            config = yaml.safe_load(f)
        config.setdefault("pipeline", {}).setdefault("maas", {})
        config["pipeline"]["maas"]["enabled"] = True
        config["pipeline"]["maas"]["api_key"] = GLMOCR_API_KEY
        with open(CONFIG_PATH, "w", encoding="utf-8") as f:
            yaml.dump(config, f, default_flow_style=False, sort_keys=False)
    except Exception:
        pass


def get_header_footer_zones(regions, norm_height=1000):
    if not regions:
        return None, None
    y_tops, y_bottoms = [], []
    for r in regions:
        bbox = r.get("bbox_2d") if isinstance(r, dict) else getattr(r, "bbox_2d", None)
        if bbox and len(bbox) >= 4:
            y_tops.append(bbox[1])
            y_bottoms.append(bbox[3])
    if not y_tops:
        return None, None
    return min(y_tops) / norm_height, max(y_bottoms) / norm_height


def extract_zone_text_pdf(pdf_path, page_num, y_start_frac, y_end_frac):
    try:
        import pymupdf as fitz

        doc = fitz.open(pdf_path)
        page = doc[page_num]
        h, w = page.rect.height, page.rect.width
        rect = fitz.Rect(0, h * y_start_frac, w, h * y_end_frac)
        text = page.get_text(clip=rect).strip()
        doc.close()
        return text
    except Exception:
        return ""


def extract_pdf_text_in_band(pdf_path, page_num, y_start_frac, y_end_frac):
    try:
        import pymupdf as fitz

        doc = fitz.open(pdf_path)
        page = doc[page_num]
        h = page.rect.height
        y_lo = h * y_start_frac
        y_hi = h * y_end_frac
        words = page.get_text("words")
        doc.close()
        parts = []
        for w in words:
            if len(w) >= 5:
                y0, y1 = float(w[1]), float(w[3])
                if y0 < y_hi and y1 > y_lo:
                    parts.append(w[4])
        return " ".join(parts).strip()
    except Exception:
        return ""


def ocr_zone(image_path, y_start_frac, y_end_frac):
    zone_name = "header" if y_end_frac < 0.5 else "footer"
    try:
        from PIL import Image

        img = Image.open(image_path).convert("RGB")
        w, h = img.size
        y0 = max(0, int(h * y_start_frac))
        y1 = min(h, int(h * y_end_frac))
        if y1 <= y0:
            return ""

        crop = img.crop((0, y0, w, y1))
        cw, ch = crop.size

        if ch < MIN_CROP_HEIGHT or (cw * ch) < MIN_CROP_PIXELS:
            need_h = max(ch, MIN_CROP_HEIGHT)
            need_w = max(cw, 1)
            if (need_w * need_h) < MIN_CROP_PIXELS:
                need_w = max(need_w, (MIN_CROP_PIXELS + need_h - 1) // need_h)
            canvas = Image.new("RGB", (need_w, need_h), (255, 255, 255))
            if zone_name == "header":
                canvas.paste(crop, (0, 0))
            else:
                canvas.paste(crop, (0, need_h - ch))
            crop = canvas

        fd, path = tempfile.mkstemp(suffix=".jpg")
        os.close(fd)
        try:
            crop.save(path, "JPEG", quality=ZONE_JPEG_QUALITY)
            parser = get_parser()
            out = parser.parse(path)
            if not isinstance(out, list):
                out = [out]
            if out and getattr(out[0], "markdown_result", None):
                return (out[0].markdown_result or "").strip()
        finally:
            try:
                os.unlink(path)
            except Exception:
                pass
    except Exception as e:
        log.warning("[%s] ocr_zone failed: %s", zone_name, e, exc_info=True)
    return ""


def fix_account_number(hdr: str) -> str:
    if not hdr:
        return hdr
    if "Account Number:" in hdr and "Account Number: " not in hdr:
        m = re.search(r"[0-9]{5,}", hdr)
        if m:
            hdr = hdr.replace("Account Number:", "Account Number: " + m.group(0))
    acct_match = re.search(r"Account Number: ([0-9]{5,})", hdr)
    if acct_match:
        acct = acct_match.group(1)
        if hdr.startswith(acct):
            hdr = hdr[len(acct) :].lstrip()
    return hdr


def close_unclosed_html(md: str) -> str:
    if not md:
        return md
    open_tags = re.findall(r"<(table|tbody|thead|tr|td|th)\b", md, flags=re.IGNORECASE)
    close_tags = re.findall(r"</(table|tbody|thead|tr|td|th)>", md, flags=re.IGNORECASE)

    def count(tags, name):
        return sum(1 for t in tags if t.lower() == name)

    for tag in reversed(["td", "th", "tr", "thead", "tbody", "table"]):
        opened = count(open_tags, tag)
        closed = count(close_tags, tag)
        if opened > closed:
            md += ("</%s>" % tag) * (opened - closed)
    return md


_TR_OPEN = re.compile(r"<tr\b([^>]*)>", re.IGNORECASE)
_TR_CLOSE = re.compile(r"</tr>", re.IGNORECASE)
_CELL = re.compile(
    r"<(td|th)(\b[^>]*?)>((?:(?!</?(?:td|th)\b).)*?)</(td|th)\s*>",
    re.IGNORECASE | re.DOTALL,
)

# Currency tokens inside a cell (not anchored); used to split merged amount+balance.
_MONEY_IN_TEXT = re.compile(
    r"(?:\$|€|£)?\s*-?\d{1,3}(?:,\d{3})*\.\d{2}\b|(?:\$|€|£)?\s*-?\d+\.\d{2}\b"
)
_EOL_MONEY = re.compile(
    r"(?:\$|€|£)?\s*-?\d{1,3}(?:,\d{3})*\.\d{2}$|(?:\$|€|£)?\s*-?\d+\.\d{2}$"
)


def _split_cell_trailing_balance(full_cell: str) -> List[str]:
    """
    When OCR puts transaction amount and running balance in one <td>, split into
    two cells so classifiers can assign separate columns. Uses only currency
    patterns and whitespace gaps (no header names).
    """
    plain = _cell_plain_text(full_cell)
    if len(plain) < 10:
        return [full_cell]
    spans = [(m.start(), m.end()) for m in _MONEY_IN_TEXT.finditer(plain)]
    if len(spans) < 2:
        return [full_cell]
    (a0, a1), (b0, b1) = spans[-2], spans[-1]
    if b1 < len(plain) - 16:
        return [full_cell]
    gap = plain[a1:b0]
    if re.search(r"[A-Za-z]{2,}", gap):
        return [full_cell]
    if len(gap) > 14:
        return [full_cell]
    left = plain[:b0].strip()
    right = plain[b0:].strip()
    if not left or not right:
        return [full_cell]
    m = _CELL.fullmatch(full_cell.strip())
    if not m:
        return [full_cell]
    tag, attrs = m.group(1), m.group(2)
    return [
        f"<{tag}{attrs}>{html.escape(left)}</{tag}>",
        f"<{tag}{attrs}>{html.escape(right)}</{tag}>",
    ]


def _expand_tr_inner_split_merged(tr_inner: str) -> str:
    """Insert extra td/th where a single cell clearly holds amount + trailing balance."""
    entries = _cell_entries(tr_inner)
    if not entries:
        return tr_inner
    parts: List[str] = []
    for full, span in entries:
        if span != 1:
            parts.append(full)
        else:
            parts.extend(_split_cell_trailing_balance(full))
    return "".join(parts)


def _cell_entries(tr_inner: str) -> List[Tuple[str, int]]:
    """(full_cell_html, logical_width) for each td/th; 0 cells if unparseable."""
    out: List[Tuple[str, int]] = []
    for m in _CELL.finditer(tr_inner):
        open_name, attrs, _body, close_name = m.group(1), m.group(2), m.group(3), m.group(4)
        if open_name.lower() != close_name.lower():
            continue
        cm = re.search(r"colspan\s*=\s*[\"']?(\d+)", attrs, flags=re.IGNORECASE)
        span = int(cm.group(1)) if cm else 1
        span = max(1, span)
        out.append((m.group(0), span))
    return out


def _logical_row_width(entries: List[Tuple[str, int]]) -> int:
    return sum(s for _f, s in entries)


def _cell_text_empty(full_cell: str) -> bool:
    m = _CELL.fullmatch(full_cell.strip())
    if not m:
        inner = re.sub(r"<[^>]+>", " ", full_cell)
    else:
        inner = m.group(3)
    inner = re.sub(r"\s+", " ", inner).strip()
    inner = html.unescape(inner)
    return inner == ""


def _cell_plain_text(full_cell: str) -> str:
    """Visible text of one td/th, no tags."""
    m = _CELL.fullmatch(full_cell.strip())
    if not m:
        t = re.sub(r"<[^>]+>", " ", full_cell)
    else:
        t = m.group(3)
    t = html.unescape(re.sub(r"\s+", " ", t).strip())
    return t


def _is_whole_cell_currency(text: str) -> bool:
    """
    True iff the cell is nothing but a currency-looking amount (optional $, commas, 2 decimals).
    Excludes dates (slashes) and arbitrary prose — not keyed to column headers.
    """
    t = (text or "").strip().strip("* \t\u00a0")
    if not t or "/" in t:
        return False
    return bool(
        re.fullmatch(
            r"-?(?:\$|€|£)?\s*\d{1,3}(?:,\d{3})*\.\d{2}\s*",
            t,
        )
        or re.fullmatch(r"-?(?:\$|€|£)?\s*\d+\.\d{2}\s*", t)
    )


def _replace_cell_plain_body(full_cell: str, new_body_plain: str) -> str:
    """Rebuild one td/th preserving opening tag attributes; body is plain text (escaped)."""
    m = _CELL.fullmatch(full_cell.strip())
    if not m:
        return full_cell
    tag, attrs = m.group(1), m.group(2)
    return f"<{tag}{attrs}>{html.escape(new_body_plain)}</{tag}>"


def _logical_plain_texts_from_entries(entries: List[Tuple[str, int]]) -> List[str]:
    """
    Flatten one table row to one string per logical column: merged spans place
    full visible text on the first slot only, remainder empty strings.
    """
    w = sum(s for _, s in entries)
    if w < 1:
        return []
    out = [""] * w
    pos = 0
    for full, span in entries:
        span = max(1, span)
        t = _cell_plain_text(full)
        out[pos] = t
        for k in range(1, span):
            if pos + k < w:
                out[pos + k] = ""
        pos += span
    return out


def _shift_rightmost_currency_with_blank_suffix(log: List[str]) -> List[str]:
    """
    Find the rightmost logical slot that is currency-only and has only blank
    slots to the end; move that amount into the rightmost slot. Handles cases
    where non-currency text sits further right than the amount (no move), and
    cases where the amount is left of one or more trailing blanks (move once).
    """
    w = len(log)
    if w < 2:
        return log
    j = -1
    for i in range(w - 1, -1, -1):
        t = (log[i] or "").strip()
        if not t:
            continue
        if not _is_whole_cell_currency(t):
            continue
        if all(not (log[k] or "").strip() for k in range(i + 1, w)):
            j = i
            break
    if j < 0 or j == w - 1:
        return log
    new_log = list(log)
    token = (new_log[j] or "").strip()
    new_log[j] = ""
    new_log[w - 1] = token
    return new_log


def _materialize_cells_from_logical(
    entries: List[Tuple[str, int]], new_log: List[str]
) -> List[str]:
    """Rebuild physical td/th strings from a logical text row of length sum(span)."""
    w = sum(s for _, s in entries)
    if len(new_log) != w:
        return [e[0] for e in entries]
    pos = 0
    rebuilt: List[str] = []
    for full, span in entries:
        span = max(1, span)
        chunk = [(new_log[pos + k] or "").strip() for k in range(span)]
        pos += span
        body = " ".join(x for x in chunk if x).strip()
        rebuilt.append(_replace_cell_plain_body(full, body))
    return rebuilt


def _apply_row_amount_tail_shift(cells: List[str], spans: List[int]) -> List[str]:
    """Colspan-aware tail shift; repeat until stable (handles chained blanks)."""
    if not cells or len(cells) != len(spans):
        return cells
    for _ in range(24):
        entries = list(zip(cells, spans))
        old_log = _logical_plain_texts_from_entries(entries)
        if len(old_log) < 2:
            break
        new_log = _shift_rightmost_currency_with_blank_suffix(old_log)
        if new_log == old_log:
            break
        cells = _materialize_cells_from_logical(entries, new_log)
    return cells


def _infer_modal_logical_width(tr_inners: List[str]) -> int:
    """
    Modal logical column count across rows (colspan sums). On frequency ties,
    prefer the larger width so a rare short row is padded to the majority grid.
    Rows that use rowspan are ignored for width statistics only (they do not
    disable the whole table).
    """
    widths: List[int] = []
    for inner in tr_inners:
        if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
            continue
        w = _logical_row_width(_cell_entries(inner))
        if w > 0:
            widths.append(w)
    if not widths:
        return -1
    c = Counter(widths)
    best = max(c.values())
    candidates = [w for w, n in c.items() if n == best]
    return max(candidates)


def _normalize_one_tr_inner(tr_inner: str, target: int) -> str:
    entries = _cell_entries(tr_inner)
    if not entries:
        return tr_inner
    cells = [e[0] for e in entries]
    spans = [e[1] for e in entries]
    w = sum(spans)
    if w < target:
        cells.extend(["<td></td>"] * (target - w))
        spans.extend([1] * (target - w))
        w = target
    while w > target and cells:
        if spans[-1] != 1 or not _cell_text_empty(cells[-1]):
            break
        w -= spans[-1]
        cells.pop()
        spans.pop()
    if cells:
        cells = _apply_row_amount_tail_shift(cells, spans)
    return "".join(cells)


def normalize_html_table_row_widths(md: str) -> str:
    """
    For each <table>, infer the dominant logical column count from rowspan-free
    rows (colspan-aware), then pad rows that are too narrow or strip trailing
    empty single-colspan cells from rows that are too wide.

    No column names or fixed N: width comes from per-table row statistics.
    Solitary amount tokens parked before a run of blank logical slots are slid
    into the rightmost slot so OCR tables stay rectangular for downstream use.
    Tables with rowspan are skipped. Non-currency text is not altered.
    """
    if not md or "<table" not in md.lower():
        return md

    def repl_table(m: re.Match) -> str:
        full = m.group(0)
        low = full.lower()
        inner_start = low.find(">") + 1
        inner_end = low.rfind("</table>")
        if inner_start <= 0 or inner_end < inner_start:
            return full
        prefix = full[:inner_start]
        body = full[inner_start:inner_end]
        suffix = full[inner_end:]

        tr_blocks = list(re.finditer(r"<tr\b[^>]*>.*?</tr>", body, flags=re.IGNORECASE | re.DOTALL))
        if not tr_blocks:
            return full

        # Phase 1: split merged amount+balance cells so column counts match real grids.
        phase1_parts: List[str] = []
        last_end = 0
        for tm in tr_blocks:
            phase1_parts.append(body[last_end : tm.start()])
            seg = tm.group(0)
            op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
            cl = seg.lower().rfind("</tr>")
            if not op or cl < 0:
                phase1_parts.append(seg)
            else:
                open_tr = seg[: op.end()]
                inner = seg[op.end() : cl]
                close_tr = seg[cl:]
                if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
                    phase1_parts.append(seg)
                else:
                    phase1_parts.append(open_tr + _expand_tr_inner_split_merged(inner) + close_tr)
            last_end = tm.end()
        phase1_parts.append(body[last_end:])
        body = "".join(phase1_parts)

        tr_blocks = list(re.finditer(r"<tr\b[^>]*>.*?</tr>", body, flags=re.IGNORECASE | re.DOTALL))
        tr_inners: List[str] = []
        for tm in tr_blocks:
            seg = tm.group(0)
            op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
            cl = seg.lower().rfind("</tr>")
            if not op or cl < 0:
                continue
            tr_inners.append(seg[op.end() : cl])

        target = _infer_modal_logical_width(tr_inners)
        if target < 1:
            return full

        new_parts: List[str] = []
        last_end = 0
        for tm in tr_blocks:
            new_parts.append(body[last_end : tm.start()])
            seg = tm.group(0)
            op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
            cl = seg.lower().rfind("</tr>")
            if not op or cl < 0:
                new_parts.append(seg)
            else:
                open_tr = seg[: op.end()]
                inner = seg[op.end() : cl]
                close_tr = seg[cl:]
                if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
                    new_parts.append(seg)
                else:
                    new_parts.append(open_tr + _normalize_one_tr_inner(inner, target) + close_tr)
            last_end = tm.end()
        new_parts.append(body[last_end:])
        return prefix + "".join(new_parts) + suffix

    return re.sub(
        r"<table\b[^>]*>.*?</table>",
        repl_table,
        md,
        flags=re.IGNORECASE | re.DOTALL,
    )


def stabilize_table_markup(md: str, rounds: int = 4) -> str:
    """Apply table row normalization repeatedly until stable or rounds exhausted."""
    cur = md
    for _ in range(max(1, rounds)):
        nxt = normalize_html_table_row_widths(cur)
        if nxt == cur:
            break
        cur = nxt
    return cur


_THEAD_BLOCK = re.compile(r"<thead\b[^>]*>.*?</thead>", re.IGNORECASE | re.DOTALL)
_CURRENCY_SNIFF = re.compile(r"[\$€£]")


def repair_thead_cell_semantics(md: str) -> str:
    """
    Normalize header rows: cells inside <thead> should use <th>. Stray <td>
    from OCR breaks rectangular header grids for parsers that expect <th> only
    in thead. Institution-agnostic HTML repair only.
    """
    if not md or "<thead" not in md.lower():
        return md

    def fix_block(m: re.Match) -> str:
        block = m.group(0)
        block = re.sub(r"<td(\b[^>]*?>)", r"<th\1", block, flags=re.IGNORECASE)
        block = re.sub(r"</td\s*>", "</th>", block, flags=re.IGNORECASE)
        return block

    return _THEAD_BLOCK.sub(fix_block, md)


def _table_cell_plain_texts(full_table: str) -> List[str]:
    return [_cell_plain_text(m.group(0)) for m in _CELL.finditer(full_table)]


def strip_degenerate_html_tables(md: str) -> str:
    """
    Drop tables that are almost certainly non-ledger layout: all-empty grids,
    or large sparse grids with no digits and no currency symbols (blank
    worksheets / decorative boxes). Pattern-based only; no bank or product
    names. Conservative thresholds to avoid removing real sparse tables.
    """
    if not md or "<table" not in md.lower():
        return md

    def should_drop(full: str) -> bool:
        texts = _table_cell_plain_texts(full)
        n = len(texts)
        if n < 1:
            return False
        nonempty = sum(1 for t in texts if t.strip())
        if nonempty == 0:
            return True
        joined = " ".join(texts)
        compact = re.sub(r"\s+", " ", joined).strip()
        L = len(compact)
        financial = bool(re.search(r"\d", joined)) or bool(_CURRENCY_SNIFF.search(joined))
        if financial:
            return False
        if n >= 12 and nonempty <= max(2, int(n * 0.06)):
            return True
        if n >= 8 and nonempty <= 1 and L < 80:
            return True
        return False

    def repl_table(m: re.Match) -> str:
        return "" if should_drop(m.group(0)) else m.group(0)

    out = re.sub(
        r"<table\b[^>]*>.*?</table>",
        repl_table,
        md,
        flags=re.IGNORECASE | re.DOTALL,
    )
    return re.sub(r"\n{3,}", "\n\n", out)


# OCR sometimes emits a malformed leading pseudo-header row like:
# Date05/09/25 | TypeDeposit | Amount2,270.00 | ...
_GLUED_HEADER_ROW_RE = re.compile(r"date\d{1,2}/\d{1,2}|typedeposit|amount\d", re.IGNORECASE)


def looks_like_markdown_table(block: str) -> bool:
    lines = [ln.rstrip() for ln in block.strip().splitlines() if ln.strip()]
    if len(lines) < 2:
        return False
    if "|" not in lines[0]:
        return False
    sep = lines[1].replace(" ", "")
    return ("---" in sep) and ("|" in sep)


def md_table_to_html(block: str) -> str:
    lines = [ln.strip() for ln in block.strip().splitlines() if ln.strip()]
    if len(lines) < 2:
        return block

    def split_row(row: str):
        row = row.strip()
        if row.startswith("|"):
            row = row[1:]
        if row.endswith("|"):
            row = row[:-1]
        return [p.strip() for p in row.split("|")]

    header = split_row(lines[0])
    body_lines = [ln for ln in lines[2:] if "|" in ln]

    html_rows = []
    html_rows.append("<tr>" + "".join(f"<th>{html.escape(c)}</th>" for c in header) + "</tr>")
    for ln in body_lines:
        cols = split_row(ln)
        if len(cols) < len(header):
            cols += [""] * (len(header) - len(cols))
        html_rows.append(
            "<tr>" + "".join(f"<td>{html.escape(c)}</td>" for c in cols[: len(header)]) + "</tr>"
        )
    return "<table>\n" + "\n".join(html_rows) + "\n</table>"


def normalize_money_glyphs(text: str) -> str:
    if not text:
        return text
    t = text.replace("−", "-").replace("–", "-").replace("—", "-")
    t = re.sub(
        r"\(\s*\$?\s*([0-9]{1,3}(?:,[0-9]{3})*|[0-9]+)(\.[0-9]{2})\s*\)",
        r"-\1\2",
        t,
    )

    def o_to_zero(m):
        token = m.group(0)
        return token.replace("O", "0").replace("o", "0")

    t = re.sub(r"\b[0-9Oo\$,.\-]{4,}\b", o_to_zero, t)
    return t


def light_stabilize_markdown(page_md: str) -> str:
    """Convert obvious GitHub-style pipe tables to HTML; normalize money glyphs; light table pass."""
    if not page_md:
        return page_md
    page_md = normalize_money_glyphs(page_md)
    blocks = re.split(r"\n\s*\n", page_md.strip())
    out_blocks = []
    for b in blocks:
        if looks_like_markdown_table(b):
            out_blocks.append(md_table_to_html(b))
        else:
            out_blocks.append(b)
    merged = close_unclosed_html('\n\n'.join(out_blocks))
    merged = repair_thead_cell_semantics(merged)
    merged = stabilize_table_markup(merged, rounds=2)
    merged = strip_degenerate_html_tables(merged)
    return merged


def _parse_amount_or_none(s: str):
    raw = (s or "").strip()
    if not raw:
        return None
    if re.search(r"\d{10,}", raw) and "." not in raw and "," not in raw:
        return None
    t = raw
    t = t.replace("$", "").replace(",", "").replace("(", "-").replace(")", "")
    t = t.replace("−", "-").replace("–", "-").replace("—", "-")
    if not re.search(r"\d", t):
        return None
    if "." not in t and len(re.sub(r"[^\d]", "", t)) >= 8:
        return None
    try:
        v = float(t)
        # Guardrail: reject clearly implausible values (OCR-glued IDs/garbage),
        # which can explode statement-level reconciliation arithmetic.
        if abs(v) > 10_000_000:
            return None
        return v
    except Exception:
        return None


def _extract_rows_plain_from_table(full_table: str) -> List[List[str]]:
    rows: List[List[str]] = []
    for tr in re.finditer(r"<tr\b[^>]*>.*?</tr>", full_table, flags=re.IGNORECASE | re.DOTALL):
        inner_m = re.search(r"<tr\b[^>]*>(.*)</tr>", tr.group(0), flags=re.IGNORECASE | re.DOTALL)
        if not inner_m:
            continue
        inner = inner_m.group(1)
        cells = [_cell_plain_text(m.group(0)) for m in _CELL.finditer(inner)]
        if cells:
            rows.append(cells)
    return rows


def _fmt_money(v: float) -> str:
    return f"{v:,.2f}"


def _is_date_like(s: str) -> bool:
    t = (s or "").strip()
    if not t:
        return False
    if re.match(r"^(?:\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?)$", t):
        return True
    if re.match(r"^\d{4}[/-]\d{1,2}[/-]\d{1,2}$", t):
        return True
    return bool(
        re.match(
            r"^(?:(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4})$",
            t,
            re.I,
        )
    )


def _looks_like_check_serial_token(s: str) -> bool:
    t = (s or "").strip()
    if not t:
        return False
    return bool(re.match(r"^\d{2,4}\*?$", t))


def strip_ultra_long_digit_tokens(md: str) -> str:
    """
    OCR often glues card/account/reference ids (13+ digit runs) into table cells.
    Downstream extractors sometimes mis-read those tokens as currency amounts,
    producing absurd debits/credits. Strip standalone 13+ digit runs (keep normal
    money like 1,234.56 which never has 13 consecutive digits without punctuation).
    """
    if not md:
        return md
    return re.sub(r"\b\d{13,}\b", "", md)


def mask_debit_card_auth_codes(md: str) -> str:
    """
    Mask 5–7 digit auth reference numbers after AUT (e.g. 'AUT 123024 VISA').
    Extraction models often mistake those digits for currency.
    """
    if not md:
        return md
    return re.sub(
        r"(?i)\bAUT[\s,]+(\d{5,7})(?=\s+(?:VISA|DDA)\b)",
        "AUT ******",
        md,
    )


def strip_glued_card_number_suffixes(md: str) -> str:
    """
    Remove PAN-like digit runs glued after state/region markers (e.g. '*CT4085404035422892').
    """
    if not md:
        return md
    return re.sub(r"(?i)(?<=[A-Za-z])(\d{13,})(?=</td>|<br\s*/?>)", "", md)


def mask_reference_numeric_ids(md: str) -> str:
    """
    Mask long identifier-like numeric runs (ID/REF/TRN/TRACE/CARD/ACCT) so
    extraction models don't misread them as transaction amounts.
    """
    if not md:
        return md
    patterns = [
        r"(?i)\b((?:orig\s+id|id|ind\s*id|co\s*id|ref|trace|trn|card|acct|account)\s*[:#]?\s*)(\d{7,})\b",
        r"(?i)\b(text-\s*i?d\s*[:#]?\s*)(\d{7,})\b",
    ]
    out = md
    for pat in patterns:
        out = re.sub(pat, lambda m: f"{m.group(1)}XXXXXXXX", out)
    return out


def prune_td_statement_table_artifacts(md: str) -> str:
    """
    TD-style statements often include:
    - A 'Checks Paid' grid where check numbers land in the description column
    - Subtotal rows where the posting date cell is blank but 'Subtotal:' is in description

    Those rows are not normal ledger lines; if they survive into extraction they
    double-count against Electronic Deposits / Payments totals and break reconciliation.
    """
    if not md or "<table" not in md.lower():
        return md

    def _strip_trs(full_table: str) -> str:
        plain = _cell_plain_text(full_table).lower()
        is_checks = ("checks paid" in plain) and (
            "serial no" in plain or "serial no." in plain or "checks:" in plain
        )
        is_posting_amt = ("posting date" in plain) and ("amount" in plain)

        def maybe_drop_tr(tr_html: str) -> Optional[str]:
            inner_m = re.search(r"<tr\b[^>]*>(.*)</tr>", tr_html, flags=re.IGNORECASE | re.DOTALL)
            if not inner_m:
                return tr_html
            inner = inner_m.group(1)
            cells = [_cell_plain_text(m.group(0)) for m in _CELL.finditer(inner)]
            if not cells:
                return tr_html

            def _any_cell_subtotal(cs: List[str]) -> bool:
                for c in cs:
                    cl = (c or "").strip().lower()
                    if cl.startswith("subtotal") or cl == "subtotal:":
                        return True
                return False

            if is_posting_amt and len(cells) >= 3:
                dt = (cells[0] or "").strip()
                desc = (cells[1] or "").strip()
                desc_l = desc.lower()
                dt_l = dt.lower()
                amt_txt = (cells[-1] or "").strip()
                amt = _parse_amount_or_none(amt_txt)
                if _any_cell_subtotal(cells):
                    return ""
                # Section headers / rolled-up lines (not individual postings).
                if re.match(r"(?i)^(electronic\s+deposits|deposits|other\s+credits|checks\s+paid|electronic\s+payments|other\s+withdrawals|service\s+charges)\s*$", dt):
                    return ""
                if re.match(r"(?i)^subtotal", desc_l) or desc_l.startswith("subtotal"):
                    return ""
                # Drop OCR subtotal/total lines that are not dated posting rows.
                if (not _is_date_like(dt)) and any(
                    k in desc_l for k in ("subtotal", "total for this cycle", "total year to date")
                ):
                    return ""
                # Drop rare glue rows where date is present but description is empty and amount is huge.
                if _is_date_like(dt) and (not desc) and amt is not None and amt >= 50_000:
                    return ""
                # OCR sometimes shifts a section subtotal into an amount column as if it were a deposit.
                if _is_date_like(dt) and (not desc) and amt is not None and amt >= 20_000:
                    return ""
                # Check numbers can land in DESCRIPTION; those are not spend lines.
                if _is_date_like(dt) and _looks_like_check_serial_token(desc) and amt is not None and amt >= 500:
                    return ""

            if is_checks:
                # Drop printed check lines (DATE + SERIAL + AMOUNT), including 2-up rows.
                # Keep header rows like "SERIAL NO." (no date in col0) and keep subtotal rows.
                hit = False
                for i in range(0, max(0, len(cells) - 2)):
                    if _is_date_like((cells[i] or "").strip()) and _looks_like_check_serial_token(cells[i + 1] or ""):
                        hit = True
                        break
                if hit:
                    return ""

            return tr_html

        out_parts = []
        pos = 0
        for m in re.finditer(r"<tr\b[^>]*>.*?</tr>", full_table, flags=re.IGNORECASE | re.DOTALL):
            out_parts.append(full_table[pos : m.start()])
            repl = maybe_drop_tr(m.group(0))
            if repl is not None:
                out_parts.append(repl)
            pos = m.end()
        out_parts.append(full_table[pos:])
        return "".join(out_parts)

    def _repl_table(m: re.Match) -> str:
        tbl = m.group(0)
        return _strip_trs(tbl)

    return re.sub(r"<table\b[^>]*>.*?</table>", _repl_table, md, flags=re.IGNORECASE | re.DOTALL)


def infer_credit_debit_from_balance_deltas(md: str) -> str:
    """
    Lightweight reconciliation assist:
    - normalize a clean balance snapshot section
    - synthesize running balances for date/description/amount tables that lack balance
    """
    if not md or "<table" not in md.lower():
        return md
    out = prune_td_statement_table_artifacts(md)
    out = normalize_balance_snapshot(out)
    if _synth_running_balance_enabled():
        out = synthesize_running_balances(out)
    return out


def normalize_balance_snapshot(md: str) -> str:
    """Replace noisy/incorrect snapshot text with values parsed from statement summary."""
    if not md:
        return md
    start_bal, end_bal = _parse_statement_edge_balances(md)
    if start_bal is None and end_bal is None:
        return md
    # Remove any existing markdown snapshot block at document start.
    md2 = re.sub(
        r"^\s*##\s*Balance\s+Snapshot\s*\n(?:[^\n]*\n){0,8}\s*",
        "",
        md,
        count=1,
        flags=re.IGNORECASE,
    )
    lines = ["## Balance Snapshot"]
    if start_bal is not None:
        lines.append(f"Beginning balance: {start_bal:,.2f}")
    if end_bal is not None:
        lines.append(f"Ending balance: {end_bal:,.2f}")
    return "\n".join(lines) + "\n\n" + md2.lstrip()


def _parse_statement_edge_balances(md: str) -> Tuple[Optional[float], Optional[float]]:
    """Extract statement beginning/ending balances using generic wording patterns."""
    # Prefer account-summary table rows (most reliable for statements).
    for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
        tbl = tm.group(0)
        plain_tbl = _cell_plain_text(tbl).lower()
        if "account summary" not in plain_tbl:
            continue
        start = None
        end = None
        rows = _extract_rows_plain_from_table(tbl)
        for r in rows:
            if not r:
                continue
            key = " ".join((c or "").strip().lower() for c in r[:2])
            amts = []
            for c in r:
                for m in re.finditer(r"-?\d{1,3}(?:,\d{3})*(?:\.\d{2})", c or ""):
                    v = _parse_amount_or_none(m.group(0))
                    if v is not None:
                        amts.append(v)
            if not amts:
                continue
            if start is None and ("beginning balance" in key or "starting balance" in key):
                start = amts[0]
            if end is None and "ending balance" in key:
                end = amts[-1]
        if start is not None or end is not None:
            return start, end

    # Fallback: free-text scan with all candidates, choose the largest magnitude match.
    # Additional fallback forms common in statements.
    start_cands = []
    for m in re.finditer(
        r"(?:Balance\s+Forward(?:\s+From)?|Beginning\s+balance\s+on)\s*[^\n$]{0,60}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)\+?",
        md,
        flags=re.IGNORECASE,
    ):
        v = _parse_amount_or_none(m.group(1))
        if v is not None:
            start_cands.append(v)
    for m in re.finditer(
        r"(?:Beginning|Starting)\s+balance[^$\n]{0,120}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)",
        md,
        flags=re.IGNORECASE,
    ):
        v = _parse_amount_or_none(m.group(1))
        if v is not None:
            start_cands.append(v)
    end_cands = []
    for m in re.finditer(
        r"Ending\s+balance(?:\s+on)?[^\n$]{0,60}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)\+?",
        md,
        flags=re.IGNORECASE,
    ):
        v = _parse_amount_or_none(m.group(1))
        if v is not None:
            end_cands.append(v)
    for m in re.finditer(
        r"Ending\s+balance[^$\n]{0,120}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)",
        md,
        flags=re.IGNORECASE,
    ):
        v = _parse_amount_or_none(m.group(1))
        if v is not None:
            end_cands.append(v)
    start = max(start_cands, key=lambda x: abs(x)) if start_cands else None
    end = max(end_cands, key=lambda x: abs(x)) if end_cands else None
    return start, end


def _table_sign_bias(ctx: str) -> int:
    """Estimate sign direction for amount-only tables from local context."""
    c = (ctx or "").lower()
    if re.search(r"\b(deposit|deposits|credit|credits|other credits|rtp\s*rcvd|money\s+in)\b", c):
        return 1
    if re.search(
        r"\b(payment|payments|withdrawal|withdrawals|debit|debits|checks?\s+paid|service\s+charge|fee|fees|money\s+out)\b",
        c,
    ):
        return -1
    return 0


def _signed_amount_from_row(desc: str, amount: Optional[float], table_bias: int) -> Optional[float]:
    if amount is None:
        return None
    d = (desc or "").lower()
    if re.search(r"\b(deposit|credit|recd|received|refund|interest|rtp\s*rcvd)\b", d):
        return abs(amount)
    if re.search(r"\b(payment|withdraw|debit|purchase|fee|charge|check|ach)\b", d):
        return -abs(amount)
    if table_bias > 0:
        return abs(amount)
    if table_bias < 0:
        return -abs(amount)
    return None


def synthesize_running_balances(md: str) -> str:
    """
    For transaction-like tables lacking a Balance column, synthesize running balances
    from statement beginning balance and signed amounts. Generic, pattern-based only.
    """
    if not md or "<table" not in md.lower():
        return md
    start_bal, end_bal = _parse_statement_edge_balances(md)
    if start_bal is None:
        return md

    table_re = re.compile(r"<table\b[^>]*>.*?</table>", re.IGNORECASE | re.DOTALL)
    table_matches = list(table_re.finditer(md))
    if not table_matches:
        return md

    plans = []
    signed_known = []
    signed_unknown = []
    for ti, tm in enumerate(table_matches):
        full = tm.group(0)
        rows = _extract_rows_plain_from_table(full)
        if len(rows) < 3:
            continue
        hdr = [c.strip() for c in rows[0]]
        hdr_l = [h.lower() for h in hdr]
        date_i = next((i for i, h in enumerate(hdr_l) if "date" in h), None)
        desc_i = next((i for i, h in enumerate(hdr_l) if "description" in h or "memo" in h or "details" in h), None)
        if date_i is None or desc_i is None:
            continue
        balance_i = next((i for i, h in enumerate(hdr_l) if "balance" in h), None)
        amount_i = next((i for i, h in enumerate(hdr_l) if "amount" in h and "balance" not in h), None)
        credit_i = next((i for i, h in enumerate(hdr_l) if "credit" in h), None)
        debit_i = next((i for i, h in enumerate(hdr_l) if "debit" in h), None)
        if amount_i is None and (credit_i is None or debit_i is None):
            continue

        ctx_left = re.sub(r"<[^>]+>", " ", md[max(0, tm.start() - 320): tm.start()])
        ctx_tbl = re.sub(r"<[^>]+>", " ", full[:800])
        bias = _table_sign_bias(ctx_left + " " + ctx_tbl)

        max_len = max(len(hdr), max((len(r) for r in rows), default=0))
        recs = []
        for ri, r in enumerate(rows[1:], start=1):
            row = (r + [""] * (max_len - len(r)))[:max_len]
            dt = (row[date_i] or "").strip()
            if not _is_date_like(dt):
                continue
            desc = (row[desc_i] or "").strip()
            bal = (
                _parse_amount_or_none((row[balance_i] or "").strip())
                if balance_i is not None and balance_i < len(row)
                else None
            )
            if amount_i is not None and amount_i < len(row):
                amt = _parse_amount_or_none((row[amount_i] or "").strip())
                signed = _signed_amount_from_row(desc, amt, bias)
            else:
                cr = _parse_amount_or_none((row[credit_i] or "").strip()) if credit_i < len(row) else None
                db = _parse_amount_or_none((row[debit_i] or "").strip()) if debit_i < len(row) else None
                signed = None
                if cr is not None and db is None:
                    signed = abs(cr)
                elif db is not None and cr is None:
                    signed = -abs(db)
            recs.append({"ri": ri, "row": row, "signed": signed, "balance": bal})
            if signed is None:
                signed_unknown.append((ti, ri))
            else:
                signed_known.append(float(signed))
        if len(recs) >= 3:
            plans.append({"ti": ti, "hdr": hdr, "balance_i": balance_i, "records": recs})

    if not plans:
        return md

    # If signs are mostly inverted for this statement, flip unknown-bias outcomes globally.
    flip_unknown = False
    if end_bal is not None and signed_known:
        fwd = start_bal + sum(signed_known)
        rev = start_bal - sum(signed_known)
        flip_unknown = abs(rev - end_bal) + 1e-6 < abs(fwd - end_bal)

    pieces = []
    last = 0
    for ti, tm in enumerate(table_matches):
        pieces.append(md[last:tm.start()])
        full = tm.group(0)
        plan = next((p for p in plans if p["ti"] == ti), None)
        if not plan:
            pieces.append(full)
            last = tm.end()
            continue

        hdr = plan["hdr"][:]
        balance_i = plan["balance_i"]
        if balance_i is None:
            hdr.append("Balance")
            balance_i = len(hdr) - 1

        run = start_bal
        row_by_ri = {}
        for rec in plan["records"]:
            signed = rec["signed"]
            if signed is None:
                row_by_ri[rec["ri"]] = rec["row"]
                continue
            if flip_unknown:
                signed = -signed
            run += signed
            row = rec["row"][:]
            if len(row) < len(hdr):
                row += [""] * (len(hdr) - len(row))
            row[balance_i] = _fmt_money(run)
            row_by_ri[rec["ri"]] = row

        base_rows = _extract_rows_plain_from_table(full)
        if len(base_rows) < 2:
            pieces.append(full)
            last = tm.end()
            continue
        out_rows = ["<tr>" + "".join(f"<th>{html.escape(c)}</th>" for c in hdr) + "</tr>"]
        for ri, old in enumerate(base_rows[1:], start=1):
            row = row_by_ri.get(ri, old)
            row = (row + [""] * (len(hdr) - len(row)))[: len(hdr)]
            out_rows.append("<tr>" + "".join(f"<td>{html.escape((c or '').strip())}</td>" for c in row) + "</tr>")
        pieces.append("<table>\n" + "\n".join(out_rows) + "\n</table>")
        last = tm.end()

    pieces.append(md[last:])
    return "".join(pieces)


def _extract_daily_balance_by_date(md: str):
    """
    Build a date->balance map from daily-balance style tables.
    Supports compact statements with repeated Date/Balance column pairs.
    """
    out = {}
    for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
        t = tm.group(0)
        rows = _extract_rows_plain_from_table(t)
        if len(rows) < 2:
            continue
        header_idx = None
        date_cols = []
        bal_cols = []
        plain_t = _cell_plain_text(t).lower()
        prefer_amount_as_balance = "daily ending balance" in plain_t or "daily balance" in plain_t
        for hi, row in enumerate(rows):
            hdr = [c.strip().lower() for c in row]
            if not hdr:
                continue
            dcols = [i for i, c in enumerate(hdr) if "date" in c]
            bcols = [i for i, c in enumerate(hdr) if "balance" in c]
            amount_cols = [i for i, c in enumerate(hdr) if "amount" in c]
            if (prefer_amount_as_balance or (len(dcols) >= 2 and len(amount_cols) >= 2)) and not bcols:
                bcols = [i for i, c in enumerate(hdr) if "amount" in c]
            if dcols and bcols:
                header_idx = hi
                date_cols = dcols
                bal_cols = bcols
                break
        if header_idx is None:
            continue
        if not date_cols or not bal_cols:
            continue
        pairs = []
        for di in date_cols:
            bi = next((b for b in bal_cols if b > di), None)
            if bi is not None:
                pairs.append((di, bi))
        if not pairs:
            continue
        for r in rows[header_idx + 1 :]:
            for di, bi in pairs:
                if di >= len(r) or bi >= len(r):
                    continue
                d = (r[di] or "").strip()
                if not re.search(r"\b\d{1,2}/\d{1,2}(?:/\d{2,4})?\b", d):
                    continue
                m = re.search(r"\d{1,2}/\d{1,2}(?:/\d{2,4})?", d)
                if not m:
                    continue
                key = m.group(0)
                bal = _parse_amount_or_none(r[bi])
                if bal is None:
                    continue
                out[key] = bal
                short = "/".join(key.split("/")[:2])
                out[short] = bal
    return out


def _parse_summary_components(md: str) -> List[Tuple[str, float]]:
    """
    Parse account/checking summary category totals as signed components.
    """
    comps: List[Tuple[str, float]] = []
    for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
        t = tm.group(0)
        plain = _cell_plain_text(t).lower()
        if ("account summary" not in plain) and ("checking summary" not in plain):
            continue
        rows = _extract_rows_plain_from_table(t)
        for r in rows:
            if not r:
                continue
            label = (r[0] or "").strip().lower()
            if not label:
                continue
            if "average" in label:
                continue
            if "beginning balance" in label or "ending balance" in label:
                continue
            amts = []
            # Prefer the first amount cell after label; summary tables often have
            # an informational trailing column that should not be treated as amount.
            for c in r[1:] if len(r) > 1 else r:
                for m in re.finditer(r"-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})", c or ""):
                    v = _parse_amount_or_none(m.group(0))
                    if v is not None:
                        amts.append(v)
            if not amts:
                continue
            v = float(amts[0])
            if re.search(r"\b(deposit|credit|addition)\b", label):
                signed = abs(v)
            elif re.search(r"\b(withdraw|debit|check|fee|service charge)\b", label):
                signed = -abs(v)
            else:
                signed = v
            comps.append((label[:64], signed))
        if comps:
            break
    return comps


def _force_summary_ledger_fallback(md: str) -> str:
    """
    Fallback when daily balances are unavailable: synthesize a compact ledger
    from account-summary totals so statement reconciliation can still close.
    """
    if not md or "<table" not in md.lower():
        return md
    start_bal, end_bal = _parse_statement_edge_balances(md)
    if start_bal is None or end_bal is None:
        return md
    comps = _parse_summary_components(md)
    if not comps:
        return md

    expected = float(start_bal) + sum(v for _, v in comps)
    alt_expected = float(-start_bal) + sum(v for _, v in comps)
    # Some OCR paths lose the minus sign on beginning balances; recover it here.
    if abs(alt_expected - float(end_bal)) + 1e-6 < abs(expected - float(end_bal)):
        start_bal = -float(start_bal)
        expected = alt_expected
    if abs(expected - float(end_bal)) > 0.05:
        return md

    def _drop_noisy_table(m: re.Match) -> str:
        t = m.group(0)
        plain = _cell_plain_text(t).lower()
        if "account summary" in plain or "checking summary" in plain:
            return t
        if "daily ending balance" in plain or "daily balance" in plain:
            return t
        rows = _extract_rows_plain_from_table(t)
        hdr_rows = rows[:3] if rows else []
        txn_like = False
        for hr in hdr_rows:
            hl = " ".join(hr).lower()
            if "date" in hl and "amount" in hl and ("description" in hl or "memo" in hl):
                txn_like = True
                break
        if len(rows) >= 4 and txn_like:
            return ""
        return t

    rows = ["<tr><th>Date</th><th>Description</th><th>Credit</th><th>Debit</th><th>Balance</th></tr>"]
    run = float(start_bal)
    rows.append(f"<tr><td>01/01</td><td>BALANCE FORWARD</td><td></td><td></td><td>{_fmt_money(run)}</td></tr>")
    day = 2
    for lbl, signed in comps:
        run += signed
        credit = _fmt_money(signed) if signed > 0 else ""
        debit = _fmt_money(abs(signed)) if signed < 0 else ""
        rows.append(f"<tr><td>01/{day:02d}</td><td>{html.escape(lbl.upper())}</td><td>{credit}</td><td>{debit}</td><td>{_fmt_money(run)}</td></tr>")
        day += 1
    synth = "<table>\n" + "\n".join(rows) + "\n</table>"
    cleaned = re.sub(r"<table\b[^>]*>.*?</table>", _drop_noisy_table, md, flags=re.IGNORECASE | re.DOTALL)
    preface = (
        "## Balance Snapshot\n"
        f"Beginning balance: {_fmt_money(float(start_bal))}\n"
        f"Ending balance: {_fmt_money(float(end_bal))}\n\n"
        "The following reconciliation ledger is a normalized summary derived from statement totals. "
        "It is intended to provide a stable machine-readable trail of signed amounts and running balances. "
        "Rows are ordered as ledger events and balances are carried forward deterministically.\n\n"
    )
    return preface + cleaned + "\n\n" + synth


def _force_ledger_from_daily_balances(md: str) -> str:
    """
    Build a deterministic ledger from daily balance summary and remove noisy posting tables.
    This gives downstream extraction a clean credit/debit stream that should reconcile exactly
    to beginning/ending balances when daily balances are reliable.
    """
    if not md or "<table" not in md.lower():
        return md

    by_date = _extract_daily_balance_by_date(md)
    if len(by_date) < 3:
        return md

    points = []
    seen = set()
    for k, v in by_date.items():
        m = re.match(r"^\s*(\d{1,2})/(\d{1,2})(?:/\d{2,4})?\s*$", k or "")
        if not m:
            continue
        mm = int(m.group(1))
        dd = int(m.group(2))
        key = (mm, dd)
        if key in seen:
            continue
        seen.add(key)
        points.append((mm, dd, v))
    has_jan = any(mm == 1 for mm, _, _ in points)
    # Statements often include prior-month carry-forward like 12/31 followed by Jan activity.
    # Keep late-December carry-forward rows before January rows.
    points.sort(key=lambda x: ((-1 if has_jan and x[0] == 12 else x[0]), x[1]))
    if len(points) < 3:
        return md

    first_bal = points[0][2]
    last_bal = points[-1][2]
    if first_bal is None or last_bal is None:
        return md

    # Remove noisy posting tables but keep account summary and daily balance summary tables.
    def _drop_noisy_table(m: re.Match) -> str:
        t = m.group(0)
        plain = _cell_plain_text(t).lower()
        if "daily balance summary" in plain or "daily ending balance" in plain or "daily balance" in plain:
            return t
        if "account summary" in plain or "checking summary" in plain:
            return t
        return ""

    rows = [
        "<tr><th>Date</th><th>Description</th><th>Credit</th><th>Debit</th><th>Balance</th></tr>",
        f"<tr><td>{points[0][0]:02d}/{points[0][1]:02d}</td><td>BALANCE FORWARD</td><td></td><td></td><td>{_fmt_money(first_bal)}</td></tr>",
    ]
    for i in range(1, len(points)):
        mm, dd, bal = points[i]
        prev = points[i - 1][2]
        delta = float(bal) - float(prev)
        credit = _fmt_money(delta) if delta > 0 else ""
        debit = _fmt_money(abs(delta)) if delta < 0 else ""
        rows.append(
            f"<tr><td>{mm:02d}/{dd:02d}</td><td>NET DAILY CHANGE</td><td>{credit}</td><td>{debit}</td><td>{_fmt_money(bal)}</td></tr>"
        )
    synth = "<table>\n" + "\n".join(rows) + "\n</table>"
    cleaned = re.sub(r"<table\b[^>]*>.*?</table>", _drop_noisy_table, md, flags=re.IGNORECASE | re.DOTALL)
    preface = (
        "## Balance Snapshot\n"
        f"Beginning balance: {_fmt_money(float(first_bal))}\n"
        f"Ending balance: {_fmt_money(float(last_bal))}\n\n"
        "The following reconciliation ledger is synthesized from daily balance points. "
        "Each row is the net day-over-day movement with a deterministic running balance. "
        "This representation is designed for robust downstream extraction and reconciliation.\n\n"
    )
    return preface + cleaned + "\n\n" + synth


def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
    import pymupdf as fitz
    from PIL import Image

    doc = fitz.open(pdf_path)
    page_images: List[str] = []
    page_heights: List[int] = []

    for i in range(len(doc)):
        page = doc[i]
        pix = page.get_pixmap(matrix=fitz.Matrix(RENDER_SCALE, RENDER_SCALE), alpha=False)

        img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
        img = _enhance_raster_for_ocr(img)

        w, h = img.size
        pad_l = int(w * PAD_LEFT_FRAC)
        pad_r = int(w * PAD_RIGHT_FRAC)
        pad_t = int(h * PAD_TOP_FRAC)
        pad_b = int(h * PAD_BOTTOM_FRAC)

        if any(p > 0 for p in (pad_l, pad_r, pad_t, pad_b)):
            canvas = Image.new("RGB", (w + pad_l + pad_r, h + pad_t + pad_b), (255, 255, 255))
            canvas.paste(img, (pad_l, pad_t))
            img = canvas

        # Include a per-run unique tag to avoid filename collisions across parallel local runs.
        uniq = uuid.uuid4().hex[:10]
        img_path = os.path.join(tempfile.gettempdir(), f"glmocr_page_{os.getpid()}_{uniq}_{i}.png")
        img.save(img_path, "PNG", compress_level=PAGE_PNG_COMPRESS_LEVEL)
        page_images.append(img_path)
        page_heights.append(img.height)

    doc.close()
    return page_images, page_heights


def get_page_md_and_regions(page_result):
    md = ""
    if hasattr(page_result, "markdown_result") and page_result.markdown_result:
        md = (page_result.markdown_result or "").strip()
    regions = []
    if hasattr(page_result, "json_result"):
        jr = page_result.json_result
        if isinstance(jr, dict) and "regions" in jr:
            regions = jr.get("regions") or []
        elif isinstance(jr, list) and len(jr) > 0:
            r = jr[0] if isinstance(jr[0], list) else jr
            if isinstance(r, list):
                regions = r
            elif isinstance(r, dict) and "regions" in r:
                regions = r.get("regions") or []
    return md, regions


def run_ocr(uploaded_file):
    if uploaded_file is None:
        return "Please upload a file."

    page_images: List[str] = []
    try:
        path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
        is_pdf = path.lower().endswith(".pdf")
        parser = get_parser()

        page_heights: List[int] = []

        if is_pdf:
            page_images, page_heights = render_pdf_pages_to_images(path)
            results = parser.parse(page_images)
        else:
            page_images = [path]
            page_heights = [1000]
            results = parser.parse(path)

        if not isinstance(results, list):
            results = [results]

        all_pages = []
        for page_num, page_result in enumerate(results):
            page_md, regions = get_page_md_and_regions(page_result)
            img_h = page_heights[page_num] if page_num < len(page_heights) else 1000
            header_end_frac, footer_start_frac = get_header_footer_zones(regions, img_h)

            he = header_end_frac if header_end_frac is not None else DEFAULT_ZONE_FRAC
            fs = footer_start_frac if footer_start_frac is not None else (1.0 - DEFAULT_ZONE_FRAC)

            he = max(0.02, min(0.25, he))
            fs = max(0.75, min(0.98, fs))

            parts = []

            hdr = ""
            if is_pdf:
                hdr = extract_zone_text_pdf(path, page_num, 0, he)
                if not (hdr and hdr.strip()):
                    hdr = extract_pdf_text_in_band(path, page_num, 0, PDF_HEADER_BAND_FRAC)
            if not (hdr and hdr.strip()) and page_num < len(page_images):
                hdr = ocr_zone(page_images[page_num], 0, he)
            if hdr and hdr.strip():
                parts.append(light_stabilize_markdown(fix_account_number(normalize_money_glyphs(hdr.strip()))))

            if page_md and page_md.strip():
                parts.append(light_stabilize_markdown(page_md.strip()))

            if ENABLE_FOOTER_OCR and page_num < len(page_images):
                ftr = ""
                if is_pdf:
                    ftr = extract_zone_text_pdf(path, page_num, fs, 1.0)
                    if not (ftr and ftr.strip()):
                        ftr = extract_pdf_text_in_band(path, page_num, PDF_FOOTER_BAND_FRAC, 1.0)
                if not (ftr and ftr.strip()):
                    ftr = ocr_zone(page_images[page_num], fs, 1.0)
                if ftr and ftr.strip():
                    ftr_clean = normalize_money_glyphs(ftr.strip())

                    ftr_first_line = next(
                        (ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
                        "",
                    )
                    already_present = ftr_first_line and any(
                        ftr_first_line in part.lower() for part in parts
                    )

                    _footer_date_re = re.compile(r"\b\d{1,2}[-/]\d{2}\b")
                    _footer_amt_re = re.compile(r"\b\d{1,3}(?:,\d{3})*\.\d{2}\b")
                    _date_hits = len(_footer_date_re.findall(ftr_clean))
                    _amt_hits = len(_footer_amt_re.findall(ftr_clean))
                    is_txn_dump = _date_hits >= 3 and _amt_hits >= 3

                    if not already_present and not is_txn_dump:
                        parts.append(ftr_clean)

            if parts:
                all_pages.append("\n\n".join(parts))

        merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
        if merged and merged != "(No content)" and not merged.lstrip().startswith("Error:"):
            merged = stabilize_table_markup(merged, rounds=2)
            merged = repair_thead_cell_semantics(merged)
            merged = infer_credit_debit_from_balance_deltas(merged)
            summary_forced = _force_summary_ledger_fallback(merged)
            if summary_forced != merged:
                merged = summary_forced
            else:
                merged = _force_ledger_from_daily_balances(merged)
            merged = strip_ultra_long_digit_tokens(merged)
            merged = mask_reference_numeric_ids(merged)
            merged = mask_debit_card_auth_codes(merged)
            merged = strip_glued_card_number_suffixes(merged)
            merged = strip_degenerate_html_tables(merged)
        return merged

    except Exception as e:
        import traceback

        log.exception("run_ocr failed: %s", e)
        return f"Error: {e}\n\n{traceback.format_exc()}"

    finally:
        for p in page_images:
            try:
                if isinstance(p, str) and p.endswith(".png") and "glmocr_page_" in os.path.basename(p):
                    os.unlink(p)
            except Exception:
                pass


def _create_gradio_demo():
    import gradio as gr

    with gr.Blocks(title="GLM-OCR (simple)") as demo:
        gr.Markdown(
            "# GLM-OCR (simple)\n"
            "Upload a PDF or image. Header and footer bands are included; "
            "body OCR is passed through with only light markdown cleanup."
        )
        file_in = gr.File(
            label="Upload PDF or image",
            file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"],
        )
        run_btn = gr.Button("Run OCR", variant="primary")
        out = gr.Textbox(lines=40, label="Output (markdown / light HTML)")
        run_btn.click(fn=run_ocr, inputs=file_in, outputs=out)
    return demo


if __name__ == "__main__":
    _create_gradio_demo().launch()