Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,61 +1,11 @@
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Simplified GLM-OCR Hugging Face / local Gradio app.
|
| 4 |
-
|
| 5 |
-
Scope (intentionally small):
|
| 6 |
-
- PDF → padded high-DPI page images → GLM-OCR body markdown
|
| 7 |
-
- Header band: PDF text extraction first, optional header OCR fallback
|
| 8 |
-
- Footer band: same pattern, with light dedup so we do not paste a full
|
| 9 |
-
transaction dump twice when the body already captured it
|
| 10 |
-
|
| 11 |
-
Universal image pipeline (same for every PDF, no keywords / no bank logic):
|
| 12 |
-
- Higher rasterization scale + extra white padding so fine print, boxed
|
| 13 |
-
section labels, and right-aligned amounts sit farther from the clip edge.
|
| 14 |
-
- Mild contrast + unsharp mask on every raster sent to the model so
|
| 15 |
-
thin rules and small glyphs are easier to read before recognition.
|
| 16 |
-
|
| 17 |
-
Explicitly omitted vs the heavy Space build:
|
| 18 |
-
- No text-layer row injection, institution-specific splits (UCB / Navy /
|
| 19 |
-
TD / First Horizon / …), or doc-wide dedupe passes.
|
| 20 |
-
|
| 21 |
-
Included (data-driven, no institution names):
|
| 22 |
-
- HTML tables: modal logical width from rowspan-free rows (colspan-aware);
|
| 23 |
-
pad short rows; trim trailing empty cells; expand each row to a logical
|
| 24 |
-
grid, then slide a solitary amount token past trailing blank logical slots
|
| 25 |
-
into the rightmost slot (colspan-aware). Rowspan rows are skipped for edits
|
| 26 |
-
but do not disable an entire table. Stabilization runs in multiple passes.
|
| 27 |
-
- Split single cells that clearly contain transaction amount + trailing balance
|
| 28 |
-
(two money tokens, tight gap) into two cells so classifiers can see a balance column.
|
| 29 |
-
- thead uses th only; degenerate empty/sparse non-financial tables are dropped.
|
| 30 |
-
|
| 31 |
-
Configure GLMOCR_API_KEY, GLM_OCR_API_KEY, or ZHIPU_API_KEY (environment). Optional: glmocr + gradio +
|
| 32 |
-
pymupdf + pillow installed.
|
| 33 |
-
"""
|
| 34 |
-
|
| 35 |
-
# Patch asyncio first (before Gradio imports it) to reduce Python 3.13 loop noise
|
| 36 |
-
import asyncio
|
| 37 |
|
| 38 |
-
try:
|
| 39 |
-
_orig_close = asyncio.BaseEventLoop.close
|
| 40 |
-
|
| 41 |
-
def _safe_close(self):
|
| 42 |
-
try:
|
| 43 |
-
_orig_close(self)
|
| 44 |
-
except (ValueError, OSError):
|
| 45 |
-
pass
|
| 46 |
-
|
| 47 |
-
asyncio.BaseEventLoop.close = _safe_close
|
| 48 |
-
except Exception:
|
| 49 |
-
pass
|
| 50 |
-
|
| 51 |
-
import html
|
| 52 |
import logging
|
| 53 |
import os
|
| 54 |
import re
|
| 55 |
import tempfile
|
| 56 |
import uuid
|
| 57 |
-
from
|
| 58 |
-
from typing import List, Optional, Tuple
|
| 59 |
|
| 60 |
import yaml
|
| 61 |
|
|
@@ -65,46 +15,27 @@ try:
|
|
| 65 |
GLMOCR_BASE = os.path.dirname(glmocr.__file__)
|
| 66 |
CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
|
| 67 |
except ImportError:
|
| 68 |
-
glmocr = None
|
| 69 |
GLMOCR_BASE = ""
|
| 70 |
CONFIG_PATH = ""
|
| 71 |
|
| 72 |
log = logging.getLogger("glmocr_simple_app")
|
| 73 |
logging.basicConfig(level=logging.INFO)
|
| 74 |
|
| 75 |
-
# ---------------------------------------------------------------------------
|
| 76 |
-
# Settings — tuned for dense financial PDFs; applies to every document
|
| 77 |
-
# ---------------------------------------------------------------------------
|
| 78 |
-
|
| 79 |
-
# Never commit secrets: Space / local runs use ZHIPU_API_KEY or GLMOCR_API_KEY.
|
| 80 |
GLMOCR_API_KEY = "cee1d52dd91a4ab591b3f6e105f8ad89.LgbQTECuzX0zrito"
|
| 81 |
if not GLMOCR_API_KEY:
|
| 82 |
log.warning(
|
| 83 |
"No ZHIPU_API_KEY or GLMOCR_API_KEY in environment; GlmOcr() will fail until you set one."
|
| 84 |
)
|
| 85 |
|
| 86 |
-
# Rasterization: higher scale = more pixels per PDF point (helps small type,
|
| 87 |
-
# boxed headers, and narrow columns). Same constant for all uploads.
|
| 88 |
RENDER_SCALE = 3.0
|
| 89 |
-
|
| 90 |
-
# White margin as a fraction of page width/height after render. Extra right
|
| 91 |
-
# margin helps right-aligned currency columns that hug the page edge.
|
| 92 |
PAD_LEFT_FRAC = 0.035
|
| 93 |
PAD_RIGHT_FRAC = 0.10
|
| 94 |
PAD_TOP_FRAC = 0.018
|
| 95 |
PAD_BOTTOM_FRAC = 0.018
|
| 96 |
|
| 97 |
-
# Synthesized running-balance columns help some heuristics but downstream LLM
|
| 98 |
-
# extractors may mis-read them as credits; default off (set GLMOCR_SYNTH_RUNNING_BALANCE=1 to enable).
|
| 99 |
-
def _synth_running_balance_enabled() -> bool:
|
| 100 |
-
return os.environ.get("GLMOCR_SYNTH_RUNNING_BALANCE", "").lower() in ("1", "true", "yes")
|
| 101 |
-
|
| 102 |
-
|
| 103 |
ENABLE_CONTRAST = True
|
| 104 |
-
# Slight contrast lift only; same factor for every file.
|
| 105 |
CONTRAST_FACTOR = 1.18
|
| 106 |
-
|
| 107 |
-
# Subtle edge enhancement after contrast (helps hairlines and small digits).
|
| 108 |
ENABLE_UNSHARP = True
|
| 109 |
UNSHARP_RADIUS = 0.78
|
| 110 |
UNSHARP_PERCENT = 76
|
|
@@ -112,29 +43,18 @@ UNSHARP_THRESHOLD = 1
|
|
| 112 |
|
| 113 |
DEFAULT_ZONE_FRAC = 0.12
|
| 114 |
PDF_HEADER_BAND_FRAC = 0.10
|
| 115 |
-
|
| 116 |
ENABLE_FOOTER_OCR = True
|
| 117 |
PDF_FOOTER_BAND_FRAC = 0.88
|
| 118 |
|
| 119 |
MIN_CROP_HEIGHT = 112
|
| 120 |
MIN_CROP_PIXELS = 112 * 112
|
| 121 |
-
|
| 122 |
-
# PNG compression 0–9; lower = less loss before GLM-OCR (same for all PDFs).
|
| 123 |
PAGE_PNG_COMPRESS_LEVEL = 3
|
| 124 |
-
# JPEG quality for small header/footer crops sent to the API.
|
| 125 |
ZONE_JPEG_QUALITY = 95
|
| 126 |
|
| 127 |
-
MIN_PDF_TEXT_CHARS_NATIVE_LAYER = 1500
|
| 128 |
-
|
| 129 |
_parser = None
|
| 130 |
|
| 131 |
|
| 132 |
def _enhance_raster_for_ocr(img):
|
| 133 |
-
"""
|
| 134 |
-
Improve legibility of every raster passed to GLM-OCR (full pages and
|
| 135 |
-
header/footer crops). No document text or keywords — same pipeline for
|
| 136 |
-
all PDFs and images.
|
| 137 |
-
"""
|
| 138 |
from PIL import ImageEnhance, ImageFilter
|
| 139 |
|
| 140 |
if ENABLE_CONTRAST:
|
|
@@ -275,1275 +195,6 @@ def ocr_zone(image_path, y_start_frac, y_end_frac):
|
|
| 275 |
return ""
|
| 276 |
|
| 277 |
|
| 278 |
-
def fix_account_number(hdr: str) -> str:
|
| 279 |
-
if not hdr:
|
| 280 |
-
return hdr
|
| 281 |
-
if "Account Number:" in hdr and "Account Number: " not in hdr:
|
| 282 |
-
m = re.search(r"[0-9]{5,}", hdr)
|
| 283 |
-
if m:
|
| 284 |
-
hdr = hdr.replace("Account Number:", "Account Number: " + m.group(0))
|
| 285 |
-
acct_match = re.search(r"Account Number: ([0-9]{5,})", hdr)
|
| 286 |
-
if acct_match:
|
| 287 |
-
acct = acct_match.group(1)
|
| 288 |
-
if hdr.startswith(acct):
|
| 289 |
-
hdr = hdr[len(acct) :].lstrip()
|
| 290 |
-
return hdr
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
def close_unclosed_html(md: str) -> str:
|
| 294 |
-
if not md:
|
| 295 |
-
return md
|
| 296 |
-
open_tags = re.findall(r"<(table|tbody|thead|tr|td|th)\b", md, flags=re.IGNORECASE)
|
| 297 |
-
close_tags = re.findall(r"</(table|tbody|thead|tr|td|th)>", md, flags=re.IGNORECASE)
|
| 298 |
-
|
| 299 |
-
def count(tags, name):
|
| 300 |
-
return sum(1 for t in tags if t.lower() == name)
|
| 301 |
-
|
| 302 |
-
for tag in reversed(["td", "th", "tr", "thead", "tbody", "table"]):
|
| 303 |
-
opened = count(open_tags, tag)
|
| 304 |
-
closed = count(close_tags, tag)
|
| 305 |
-
if opened > closed:
|
| 306 |
-
md += ("</%s>" % tag) * (opened - closed)
|
| 307 |
-
return md
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
_TR_OPEN = re.compile(r"<tr\b([^>]*)>", re.IGNORECASE)
|
| 311 |
-
_TR_CLOSE = re.compile(r"</tr>", re.IGNORECASE)
|
| 312 |
-
_CELL = re.compile(
|
| 313 |
-
r"<(td|th)(\b[^>]*?)>((?:(?!</?(?:td|th)\b).)*?)</(td|th)\s*>",
|
| 314 |
-
re.IGNORECASE | re.DOTALL,
|
| 315 |
-
)
|
| 316 |
-
|
| 317 |
-
# Currency tokens inside a cell (not anchored); used to split merged amount+balance.
|
| 318 |
-
_MONEY_IN_TEXT = re.compile(
|
| 319 |
-
r"(?:\$|€|£)?\s*-?\d{1,3}(?:,\d{3})*\.\d{2}\b|(?:\$|€|£)?\s*-?\d+\.\d{2}\b"
|
| 320 |
-
)
|
| 321 |
-
_EOL_MONEY = re.compile(
|
| 322 |
-
r"(?:\$|€|£)?\s*-?\d{1,3}(?:,\d{3})*\.\d{2}$|(?:\$|€|£)?\s*-?\d+\.\d{2}$"
|
| 323 |
-
)
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
def _split_cell_trailing_balance(full_cell: str) -> List[str]:
|
| 327 |
-
"""
|
| 328 |
-
When OCR puts transaction amount and running balance in one <td>, split into
|
| 329 |
-
two cells so classifiers can assign separate columns. Uses only currency
|
| 330 |
-
patterns and whitespace gaps (no header names).
|
| 331 |
-
"""
|
| 332 |
-
plain = _cell_plain_text(full_cell)
|
| 333 |
-
if len(plain) < 10:
|
| 334 |
-
return [full_cell]
|
| 335 |
-
spans = [(m.start(), m.end()) for m in _MONEY_IN_TEXT.finditer(plain)]
|
| 336 |
-
if len(spans) < 2:
|
| 337 |
-
return [full_cell]
|
| 338 |
-
(a0, a1), (b0, b1) = spans[-2], spans[-1]
|
| 339 |
-
if b1 < len(plain) - 16:
|
| 340 |
-
return [full_cell]
|
| 341 |
-
gap = plain[a1:b0]
|
| 342 |
-
if re.search(r"[A-Za-z]{2,}", gap):
|
| 343 |
-
return [full_cell]
|
| 344 |
-
if len(gap) > 14:
|
| 345 |
-
return [full_cell]
|
| 346 |
-
left = plain[:b0].strip()
|
| 347 |
-
right = plain[b0:].strip()
|
| 348 |
-
if not left or not right:
|
| 349 |
-
return [full_cell]
|
| 350 |
-
m = _CELL.fullmatch(full_cell.strip())
|
| 351 |
-
if not m:
|
| 352 |
-
return [full_cell]
|
| 353 |
-
tag, attrs = m.group(1), m.group(2)
|
| 354 |
-
return [
|
| 355 |
-
f"<{tag}{attrs}>{html.escape(left)}</{tag}>",
|
| 356 |
-
f"<{tag}{attrs}>{html.escape(right)}</{tag}>",
|
| 357 |
-
]
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
def _expand_tr_inner_split_merged(tr_inner: str) -> str:
|
| 361 |
-
"""Insert extra td/th where a single cell clearly holds amount + trailing balance."""
|
| 362 |
-
entries = _cell_entries(tr_inner)
|
| 363 |
-
if not entries:
|
| 364 |
-
return tr_inner
|
| 365 |
-
parts: List[str] = []
|
| 366 |
-
for full, span in entries:
|
| 367 |
-
if span != 1:
|
| 368 |
-
parts.append(full)
|
| 369 |
-
else:
|
| 370 |
-
parts.extend(_split_cell_trailing_balance(full))
|
| 371 |
-
return "".join(parts)
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
def _cell_entries(tr_inner: str) -> List[Tuple[str, int]]:
|
| 375 |
-
"""(full_cell_html, logical_width) for each td/th; 0 cells if unparseable."""
|
| 376 |
-
out: List[Tuple[str, int]] = []
|
| 377 |
-
for m in _CELL.finditer(tr_inner):
|
| 378 |
-
open_name, attrs, _body, close_name = m.group(1), m.group(2), m.group(3), m.group(4)
|
| 379 |
-
if open_name.lower() != close_name.lower():
|
| 380 |
-
continue
|
| 381 |
-
cm = re.search(r"colspan\s*=\s*[\"']?(\d+)", attrs, flags=re.IGNORECASE)
|
| 382 |
-
span = int(cm.group(1)) if cm else 1
|
| 383 |
-
span = max(1, span)
|
| 384 |
-
out.append((m.group(0), span))
|
| 385 |
-
return out
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
def _logical_row_width(entries: List[Tuple[str, int]]) -> int:
|
| 389 |
-
return sum(s for _f, s in entries)
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
def _cell_text_empty(full_cell: str) -> bool:
|
| 393 |
-
m = _CELL.fullmatch(full_cell.strip())
|
| 394 |
-
if not m:
|
| 395 |
-
inner = re.sub(r"<[^>]+>", " ", full_cell)
|
| 396 |
-
else:
|
| 397 |
-
inner = m.group(3)
|
| 398 |
-
inner = re.sub(r"\s+", " ", inner).strip()
|
| 399 |
-
inner = html.unescape(inner)
|
| 400 |
-
return inner == ""
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
def _cell_plain_text(full_cell: str) -> str:
|
| 404 |
-
"""Visible text of one td/th, no tags."""
|
| 405 |
-
m = _CELL.fullmatch(full_cell.strip())
|
| 406 |
-
if not m:
|
| 407 |
-
t = re.sub(r"<[^>]+>", " ", full_cell)
|
| 408 |
-
else:
|
| 409 |
-
t = m.group(3)
|
| 410 |
-
t = html.unescape(re.sub(r"\s+", " ", t).strip())
|
| 411 |
-
return t
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
def _is_whole_cell_currency(text: str) -> bool:
|
| 415 |
-
"""
|
| 416 |
-
True iff the cell is nothing but a currency-looking amount (optional $, commas, 2 decimals).
|
| 417 |
-
Excludes dates (slashes) and arbitrary prose — not keyed to column headers.
|
| 418 |
-
"""
|
| 419 |
-
t = (text or "").strip().strip("* \t\u00a0")
|
| 420 |
-
if not t or "/" in t:
|
| 421 |
-
return False
|
| 422 |
-
return bool(
|
| 423 |
-
re.fullmatch(
|
| 424 |
-
r"-?(?:\$|€|£)?\s*\d{1,3}(?:,\d{3})*\.\d{2}\s*",
|
| 425 |
-
t,
|
| 426 |
-
)
|
| 427 |
-
or re.fullmatch(r"-?(?:\$|€|£)?\s*\d+\.\d{2}\s*", t)
|
| 428 |
-
)
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
def _replace_cell_plain_body(full_cell: str, new_body_plain: str) -> str:
|
| 432 |
-
"""Rebuild one td/th preserving opening tag attributes; body is plain text (escaped)."""
|
| 433 |
-
m = _CELL.fullmatch(full_cell.strip())
|
| 434 |
-
if not m:
|
| 435 |
-
return full_cell
|
| 436 |
-
tag, attrs = m.group(1), m.group(2)
|
| 437 |
-
return f"<{tag}{attrs}>{html.escape(new_body_plain)}</{tag}>"
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
def _logical_plain_texts_from_entries(entries: List[Tuple[str, int]]) -> List[str]:
|
| 441 |
-
"""
|
| 442 |
-
Flatten one table row to one string per logical column: merged spans place
|
| 443 |
-
full visible text on the first slot only, remainder empty strings.
|
| 444 |
-
"""
|
| 445 |
-
w = sum(s for _, s in entries)
|
| 446 |
-
if w < 1:
|
| 447 |
-
return []
|
| 448 |
-
out = [""] * w
|
| 449 |
-
pos = 0
|
| 450 |
-
for full, span in entries:
|
| 451 |
-
span = max(1, span)
|
| 452 |
-
t = _cell_plain_text(full)
|
| 453 |
-
out[pos] = t
|
| 454 |
-
for k in range(1, span):
|
| 455 |
-
if pos + k < w:
|
| 456 |
-
out[pos + k] = ""
|
| 457 |
-
pos += span
|
| 458 |
-
return out
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
def _shift_rightmost_currency_with_blank_suffix(log: List[str]) -> List[str]:
|
| 462 |
-
"""
|
| 463 |
-
Find the rightmost logical slot that is currency-only and has only blank
|
| 464 |
-
slots to the end; move that amount into the rightmost slot. Handles cases
|
| 465 |
-
where non-currency text sits further right than the amount (no move), and
|
| 466 |
-
cases where the amount is left of one or more trailing blanks (move once).
|
| 467 |
-
"""
|
| 468 |
-
w = len(log)
|
| 469 |
-
if w < 2:
|
| 470 |
-
return log
|
| 471 |
-
j = -1
|
| 472 |
-
for i in range(w - 1, -1, -1):
|
| 473 |
-
t = (log[i] or "").strip()
|
| 474 |
-
if not t:
|
| 475 |
-
continue
|
| 476 |
-
if not _is_whole_cell_currency(t):
|
| 477 |
-
continue
|
| 478 |
-
if all(not (log[k] or "").strip() for k in range(i + 1, w)):
|
| 479 |
-
j = i
|
| 480 |
-
break
|
| 481 |
-
if j < 0 or j == w - 1:
|
| 482 |
-
return log
|
| 483 |
-
new_log = list(log)
|
| 484 |
-
token = (new_log[j] or "").strip()
|
| 485 |
-
new_log[j] = ""
|
| 486 |
-
new_log[w - 1] = token
|
| 487 |
-
return new_log
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
def _materialize_cells_from_logical(
|
| 491 |
-
entries: List[Tuple[str, int]], new_log: List[str]
|
| 492 |
-
) -> List[str]:
|
| 493 |
-
"""Rebuild physical td/th strings from a logical text row of length sum(span)."""
|
| 494 |
-
w = sum(s for _, s in entries)
|
| 495 |
-
if len(new_log) != w:
|
| 496 |
-
return [e[0] for e in entries]
|
| 497 |
-
pos = 0
|
| 498 |
-
rebuilt: List[str] = []
|
| 499 |
-
for full, span in entries:
|
| 500 |
-
span = max(1, span)
|
| 501 |
-
chunk = [(new_log[pos + k] or "").strip() for k in range(span)]
|
| 502 |
-
pos += span
|
| 503 |
-
body = " ".join(x for x in chunk if x).strip()
|
| 504 |
-
rebuilt.append(_replace_cell_plain_body(full, body))
|
| 505 |
-
return rebuilt
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
def _apply_row_amount_tail_shift(cells: List[str], spans: List[int]) -> List[str]:
|
| 509 |
-
"""Colspan-aware tail shift; repeat until stable (handles chained blanks)."""
|
| 510 |
-
if not cells or len(cells) != len(spans):
|
| 511 |
-
return cells
|
| 512 |
-
for _ in range(24):
|
| 513 |
-
entries = list(zip(cells, spans))
|
| 514 |
-
old_log = _logical_plain_texts_from_entries(entries)
|
| 515 |
-
if len(old_log) < 2:
|
| 516 |
-
break
|
| 517 |
-
new_log = _shift_rightmost_currency_with_blank_suffix(old_log)
|
| 518 |
-
if new_log == old_log:
|
| 519 |
-
break
|
| 520 |
-
cells = _materialize_cells_from_logical(entries, new_log)
|
| 521 |
-
return cells
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
def _infer_modal_logical_width(tr_inners: List[str]) -> int:
|
| 525 |
-
"""
|
| 526 |
-
Modal logical column count across rows (colspan sums). On frequency ties,
|
| 527 |
-
prefer the larger width so a rare short row is padded to the majority grid.
|
| 528 |
-
Rows that use rowspan are ignored for width statistics only (they do not
|
| 529 |
-
disable the whole table).
|
| 530 |
-
"""
|
| 531 |
-
widths: List[int] = []
|
| 532 |
-
for inner in tr_inners:
|
| 533 |
-
if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
|
| 534 |
-
continue
|
| 535 |
-
w = _logical_row_width(_cell_entries(inner))
|
| 536 |
-
if w > 0:
|
| 537 |
-
widths.append(w)
|
| 538 |
-
if not widths:
|
| 539 |
-
return -1
|
| 540 |
-
c = Counter(widths)
|
| 541 |
-
best = max(c.values())
|
| 542 |
-
candidates = [w for w, n in c.items() if n == best]
|
| 543 |
-
return max(candidates)
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
def _normalize_one_tr_inner(tr_inner: str, target: int) -> str:
|
| 547 |
-
entries = _cell_entries(tr_inner)
|
| 548 |
-
if not entries:
|
| 549 |
-
return tr_inner
|
| 550 |
-
cells = [e[0] for e in entries]
|
| 551 |
-
spans = [e[1] for e in entries]
|
| 552 |
-
w = sum(spans)
|
| 553 |
-
if w < target:
|
| 554 |
-
cells.extend(["<td></td>"] * (target - w))
|
| 555 |
-
spans.extend([1] * (target - w))
|
| 556 |
-
w = target
|
| 557 |
-
while w > target and cells:
|
| 558 |
-
if spans[-1] != 1 or not _cell_text_empty(cells[-1]):
|
| 559 |
-
break
|
| 560 |
-
w -= spans[-1]
|
| 561 |
-
cells.pop()
|
| 562 |
-
spans.pop()
|
| 563 |
-
if cells:
|
| 564 |
-
cells = _apply_row_amount_tail_shift(cells, spans)
|
| 565 |
-
return "".join(cells)
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
def normalize_html_table_row_widths(md: str) -> str:
|
| 569 |
-
"""
|
| 570 |
-
For each <table>, infer the dominant logical column count from rowspan-free
|
| 571 |
-
rows (colspan-aware), then pad rows that are too narrow or strip trailing
|
| 572 |
-
empty single-colspan cells from rows that are too wide.
|
| 573 |
-
|
| 574 |
-
No column names or fixed N: width comes from per-table row statistics.
|
| 575 |
-
Solitary amount tokens parked before a run of blank logical slots are slid
|
| 576 |
-
into the rightmost slot so OCR tables stay rectangular for downstream use.
|
| 577 |
-
Tables with rowspan are skipped. Non-currency text is not altered.
|
| 578 |
-
"""
|
| 579 |
-
if not md or "<table" not in md.lower():
|
| 580 |
-
return md
|
| 581 |
-
|
| 582 |
-
def repl_table(m: re.Match) -> str:
|
| 583 |
-
full = m.group(0)
|
| 584 |
-
low = full.lower()
|
| 585 |
-
inner_start = low.find(">") + 1
|
| 586 |
-
inner_end = low.rfind("</table>")
|
| 587 |
-
if inner_start <= 0 or inner_end < inner_start:
|
| 588 |
-
return full
|
| 589 |
-
prefix = full[:inner_start]
|
| 590 |
-
body = full[inner_start:inner_end]
|
| 591 |
-
suffix = full[inner_end:]
|
| 592 |
-
|
| 593 |
-
tr_blocks = list(re.finditer(r"<tr\b[^>]*>.*?</tr>", body, flags=re.IGNORECASE | re.DOTALL))
|
| 594 |
-
if not tr_blocks:
|
| 595 |
-
return full
|
| 596 |
-
|
| 597 |
-
# Phase 1: split merged amount+balance cells so column counts match real grids.
|
| 598 |
-
phase1_parts: List[str] = []
|
| 599 |
-
last_end = 0
|
| 600 |
-
for tm in tr_blocks:
|
| 601 |
-
phase1_parts.append(body[last_end : tm.start()])
|
| 602 |
-
seg = tm.group(0)
|
| 603 |
-
op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
|
| 604 |
-
cl = seg.lower().rfind("</tr>")
|
| 605 |
-
if not op or cl < 0:
|
| 606 |
-
phase1_parts.append(seg)
|
| 607 |
-
else:
|
| 608 |
-
open_tr = seg[: op.end()]
|
| 609 |
-
inner = seg[op.end() : cl]
|
| 610 |
-
close_tr = seg[cl:]
|
| 611 |
-
if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
|
| 612 |
-
phase1_parts.append(seg)
|
| 613 |
-
else:
|
| 614 |
-
phase1_parts.append(open_tr + _expand_tr_inner_split_merged(inner) + close_tr)
|
| 615 |
-
last_end = tm.end()
|
| 616 |
-
phase1_parts.append(body[last_end:])
|
| 617 |
-
body = "".join(phase1_parts)
|
| 618 |
-
|
| 619 |
-
tr_blocks = list(re.finditer(r"<tr\b[^>]*>.*?</tr>", body, flags=re.IGNORECASE | re.DOTALL))
|
| 620 |
-
tr_inners: List[str] = []
|
| 621 |
-
for tm in tr_blocks:
|
| 622 |
-
seg = tm.group(0)
|
| 623 |
-
op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
|
| 624 |
-
cl = seg.lower().rfind("</tr>")
|
| 625 |
-
if not op or cl < 0:
|
| 626 |
-
continue
|
| 627 |
-
tr_inners.append(seg[op.end() : cl])
|
| 628 |
-
|
| 629 |
-
target = _infer_modal_logical_width(tr_inners)
|
| 630 |
-
if target < 1:
|
| 631 |
-
return full
|
| 632 |
-
|
| 633 |
-
new_parts: List[str] = []
|
| 634 |
-
last_end = 0
|
| 635 |
-
for tm in tr_blocks:
|
| 636 |
-
new_parts.append(body[last_end : tm.start()])
|
| 637 |
-
seg = tm.group(0)
|
| 638 |
-
op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
|
| 639 |
-
cl = seg.lower().rfind("</tr>")
|
| 640 |
-
if not op or cl < 0:
|
| 641 |
-
new_parts.append(seg)
|
| 642 |
-
else:
|
| 643 |
-
open_tr = seg[: op.end()]
|
| 644 |
-
inner = seg[op.end() : cl]
|
| 645 |
-
close_tr = seg[cl:]
|
| 646 |
-
if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
|
| 647 |
-
new_parts.append(seg)
|
| 648 |
-
else:
|
| 649 |
-
new_parts.append(open_tr + _normalize_one_tr_inner(inner, target) + close_tr)
|
| 650 |
-
last_end = tm.end()
|
| 651 |
-
new_parts.append(body[last_end:])
|
| 652 |
-
return prefix + "".join(new_parts) + suffix
|
| 653 |
-
|
| 654 |
-
return re.sub(
|
| 655 |
-
r"<table\b[^>]*>.*?</table>",
|
| 656 |
-
repl_table,
|
| 657 |
-
md,
|
| 658 |
-
flags=re.IGNORECASE | re.DOTALL,
|
| 659 |
-
)
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
def stabilize_table_markup(md: str, rounds: int = 4) -> str:
|
| 663 |
-
"""Apply table row normalization repeatedly until stable or rounds exhausted."""
|
| 664 |
-
cur = md
|
| 665 |
-
for _ in range(max(1, rounds)):
|
| 666 |
-
nxt = normalize_html_table_row_widths(cur)
|
| 667 |
-
if nxt == cur:
|
| 668 |
-
break
|
| 669 |
-
cur = nxt
|
| 670 |
-
return cur
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
_THEAD_BLOCK = re.compile(r"<thead\b[^>]*>.*?</thead>", re.IGNORECASE | re.DOTALL)
|
| 674 |
-
_CURRENCY_SNIFF = re.compile(r"[\$€£]")
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
def repair_thead_cell_semantics(md: str) -> str:
|
| 678 |
-
"""
|
| 679 |
-
Normalize header rows: cells inside <thead> should use <th>. Stray <td>
|
| 680 |
-
from OCR breaks rectangular header grids for parsers that expect <th> only
|
| 681 |
-
in thead. Institution-agnostic HTML repair only.
|
| 682 |
-
"""
|
| 683 |
-
if not md or "<thead" not in md.lower():
|
| 684 |
-
return md
|
| 685 |
-
|
| 686 |
-
def fix_block(m: re.Match) -> str:
|
| 687 |
-
block = m.group(0)
|
| 688 |
-
block = re.sub(r"<td(\b[^>]*?>)", r"<th\1", block, flags=re.IGNORECASE)
|
| 689 |
-
block = re.sub(r"</td\s*>", "</th>", block, flags=re.IGNORECASE)
|
| 690 |
-
return block
|
| 691 |
-
|
| 692 |
-
return _THEAD_BLOCK.sub(fix_block, md)
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
def _table_cell_plain_texts(full_table: str) -> List[str]:
|
| 696 |
-
return [_cell_plain_text(m.group(0)) for m in _CELL.finditer(full_table)]
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
def strip_degenerate_html_tables(md: str) -> str:
|
| 700 |
-
"""
|
| 701 |
-
Drop tables that are almost certainly non-ledger layout: all-empty grids,
|
| 702 |
-
or large sparse grids with no digits and no currency symbols (blank
|
| 703 |
-
worksheets / decorative boxes). Pattern-based only; no bank or product
|
| 704 |
-
names. Conservative thresholds to avoid removing real sparse tables.
|
| 705 |
-
"""
|
| 706 |
-
if not md or "<table" not in md.lower():
|
| 707 |
-
return md
|
| 708 |
-
|
| 709 |
-
def should_drop(full: str) -> bool:
|
| 710 |
-
texts = _table_cell_plain_texts(full)
|
| 711 |
-
n = len(texts)
|
| 712 |
-
if n < 1:
|
| 713 |
-
return False
|
| 714 |
-
nonempty = sum(1 for t in texts if t.strip())
|
| 715 |
-
if nonempty == 0:
|
| 716 |
-
return True
|
| 717 |
-
joined = " ".join(texts)
|
| 718 |
-
compact = re.sub(r"\s+", " ", joined).strip()
|
| 719 |
-
L = len(compact)
|
| 720 |
-
financial = bool(re.search(r"\d", joined)) or bool(_CURRENCY_SNIFF.search(joined))
|
| 721 |
-
if financial:
|
| 722 |
-
return False
|
| 723 |
-
if n >= 12 and nonempty <= max(2, int(n * 0.06)):
|
| 724 |
-
return True
|
| 725 |
-
if n >= 8 and nonempty <= 1 and L < 80:
|
| 726 |
-
return True
|
| 727 |
-
return False
|
| 728 |
-
|
| 729 |
-
def repl_table(m: re.Match) -> str:
|
| 730 |
-
return "" if should_drop(m.group(0)) else m.group(0)
|
| 731 |
-
|
| 732 |
-
out = re.sub(
|
| 733 |
-
r"<table\b[^>]*>.*?</table>",
|
| 734 |
-
repl_table,
|
| 735 |
-
md,
|
| 736 |
-
flags=re.IGNORECASE | re.DOTALL,
|
| 737 |
-
)
|
| 738 |
-
return re.sub(r"\n{3,}", "\n\n", out)
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
# OCR sometimes emits a malformed leading pseudo-header row like:
|
| 742 |
-
# Date05/09/25 | TypeDeposit | Amount2,270.00 | ...
|
| 743 |
-
_GLUED_HEADER_ROW_RE = re.compile(r"date\d{1,2}/\d{1,2}|typedeposit|amount\d", re.IGNORECASE)
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
def looks_like_markdown_table(block: str) -> bool:
|
| 747 |
-
lines = [ln.rstrip() for ln in block.strip().splitlines() if ln.strip()]
|
| 748 |
-
if len(lines) < 2:
|
| 749 |
-
return False
|
| 750 |
-
if "|" not in lines[0]:
|
| 751 |
-
return False
|
| 752 |
-
sep = lines[1].replace(" ", "")
|
| 753 |
-
return ("---" in sep) and ("|" in sep)
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
def md_table_to_html(block: str) -> str:
|
| 757 |
-
lines = [ln.strip() for ln in block.strip().splitlines() if ln.strip()]
|
| 758 |
-
if len(lines) < 2:
|
| 759 |
-
return block
|
| 760 |
-
|
| 761 |
-
def split_row(row: str):
|
| 762 |
-
row = row.strip()
|
| 763 |
-
if row.startswith("|"):
|
| 764 |
-
row = row[1:]
|
| 765 |
-
if row.endswith("|"):
|
| 766 |
-
row = row[:-1]
|
| 767 |
-
return [p.strip() for p in row.split("|")]
|
| 768 |
-
|
| 769 |
-
header = split_row(lines[0])
|
| 770 |
-
body_lines = [ln for ln in lines[2:] if "|" in ln]
|
| 771 |
-
|
| 772 |
-
html_rows = []
|
| 773 |
-
html_rows.append("<tr>" + "".join(f"<th>{html.escape(c)}</th>" for c in header) + "</tr>")
|
| 774 |
-
for ln in body_lines:
|
| 775 |
-
cols = split_row(ln)
|
| 776 |
-
if len(cols) < len(header):
|
| 777 |
-
cols += [""] * (len(header) - len(cols))
|
| 778 |
-
html_rows.append(
|
| 779 |
-
"<tr>" + "".join(f"<td>{html.escape(c)}</td>" for c in cols[: len(header)]) + "</tr>"
|
| 780 |
-
)
|
| 781 |
-
return "<table>\n" + "\n".join(html_rows) + "\n</table>"
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
def normalize_money_glyphs(text: str) -> str:
|
| 785 |
-
if not text:
|
| 786 |
-
return text
|
| 787 |
-
t = text.replace("−", "-").replace("–", "-").replace("—", "-")
|
| 788 |
-
t = re.sub(
|
| 789 |
-
r"\(\s*\$?\s*([0-9]{1,3}(?:,[0-9]{3})*|[0-9]+)(\.[0-9]{2})\s*\)",
|
| 790 |
-
r"-\1\2",
|
| 791 |
-
t,
|
| 792 |
-
)
|
| 793 |
-
|
| 794 |
-
def o_to_zero(m):
|
| 795 |
-
token = m.group(0)
|
| 796 |
-
return token.replace("O", "0").replace("o", "0")
|
| 797 |
-
|
| 798 |
-
t = re.sub(r"\b[0-9Oo\$,.\-]{4,}\b", o_to_zero, t)
|
| 799 |
-
return t
|
| 800 |
-
|
| 801 |
-
|
| 802 |
-
def light_stabilize_markdown(page_md: str) -> str:
|
| 803 |
-
"""Convert obvious GitHub-style pipe tables to HTML; normalize money glyphs; light table pass."""
|
| 804 |
-
if not page_md:
|
| 805 |
-
return page_md
|
| 806 |
-
page_md = normalize_money_glyphs(page_md)
|
| 807 |
-
blocks = re.split(r"\n\s*\n", page_md.strip())
|
| 808 |
-
out_blocks = []
|
| 809 |
-
for b in blocks:
|
| 810 |
-
if looks_like_markdown_table(b):
|
| 811 |
-
out_blocks.append(md_table_to_html(b))
|
| 812 |
-
else:
|
| 813 |
-
out_blocks.append(b)
|
| 814 |
-
merged = close_unclosed_html('\n\n'.join(out_blocks))
|
| 815 |
-
merged = repair_thead_cell_semantics(merged)
|
| 816 |
-
merged = stabilize_table_markup(merged, rounds=2)
|
| 817 |
-
merged = strip_degenerate_html_tables(merged)
|
| 818 |
-
return merged
|
| 819 |
-
|
| 820 |
-
|
| 821 |
-
def _parse_amount_or_none(s: str):
|
| 822 |
-
raw = (s or "").strip()
|
| 823 |
-
if not raw:
|
| 824 |
-
return None
|
| 825 |
-
if re.search(r"\d{10,}", raw) and "." not in raw and "," not in raw:
|
| 826 |
-
return None
|
| 827 |
-
t = raw
|
| 828 |
-
t = t.replace("$", "").replace(",", "").replace("(", "-").replace(")", "")
|
| 829 |
-
t = t.replace("−", "-").replace("–", "-").replace("—", "-")
|
| 830 |
-
if not re.search(r"\d", t):
|
| 831 |
-
return None
|
| 832 |
-
if "." not in t and len(re.sub(r"[^\d]", "", t)) >= 8:
|
| 833 |
-
return None
|
| 834 |
-
try:
|
| 835 |
-
v = float(t)
|
| 836 |
-
# Guardrail: reject clearly implausible values (OCR-glued IDs/garbage),
|
| 837 |
-
# which can explode statement-level reconciliation arithmetic.
|
| 838 |
-
if abs(v) > 10_000_000:
|
| 839 |
-
return None
|
| 840 |
-
return v
|
| 841 |
-
except Exception:
|
| 842 |
-
return None
|
| 843 |
-
|
| 844 |
-
|
| 845 |
-
def _extract_rows_plain_from_table(full_table: str) -> List[List[str]]:
|
| 846 |
-
rows: List[List[str]] = []
|
| 847 |
-
for tr in re.finditer(r"<tr\b[^>]*>.*?</tr>", full_table, flags=re.IGNORECASE | re.DOTALL):
|
| 848 |
-
inner_m = re.search(r"<tr\b[^>]*>(.*)</tr>", tr.group(0), flags=re.IGNORECASE | re.DOTALL)
|
| 849 |
-
if not inner_m:
|
| 850 |
-
continue
|
| 851 |
-
inner = inner_m.group(1)
|
| 852 |
-
cells = [_cell_plain_text(m.group(0)) for m in _CELL.finditer(inner)]
|
| 853 |
-
if cells:
|
| 854 |
-
rows.append(cells)
|
| 855 |
-
return rows
|
| 856 |
-
|
| 857 |
-
|
| 858 |
-
def _fmt_money(v: float) -> str:
|
| 859 |
-
return f"{v:,.2f}"
|
| 860 |
-
|
| 861 |
-
|
| 862 |
-
def _is_date_like(s: str) -> bool:
|
| 863 |
-
t = (s or "").strip()
|
| 864 |
-
if not t:
|
| 865 |
-
return False
|
| 866 |
-
if re.match(r"^(?:\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?)$", t):
|
| 867 |
-
return True
|
| 868 |
-
if re.match(r"^\d{4}[/-]\d{1,2}[/-]\d{1,2}$", t):
|
| 869 |
-
return True
|
| 870 |
-
return bool(
|
| 871 |
-
re.match(
|
| 872 |
-
r"^(?:(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4})$",
|
| 873 |
-
t,
|
| 874 |
-
re.I,
|
| 875 |
-
)
|
| 876 |
-
)
|
| 877 |
-
|
| 878 |
-
|
| 879 |
-
def _looks_like_check_serial_token(s: str) -> bool:
|
| 880 |
-
t = (s or "").strip()
|
| 881 |
-
if not t:
|
| 882 |
-
return False
|
| 883 |
-
return bool(re.match(r"^\d{2,4}\*?$", t))
|
| 884 |
-
|
| 885 |
-
|
| 886 |
-
def strip_ultra_long_digit_tokens(md: str) -> str:
|
| 887 |
-
"""
|
| 888 |
-
OCR often glues card/account/reference ids (13+ digit runs) into table cells.
|
| 889 |
-
Downstream extractors sometimes mis-read those tokens as currency amounts,
|
| 890 |
-
producing absurd debits/credits. Strip standalone 13+ digit runs (keep normal
|
| 891 |
-
money like 1,234.56 which never has 13 consecutive digits without punctuation).
|
| 892 |
-
"""
|
| 893 |
-
if not md:
|
| 894 |
-
return md
|
| 895 |
-
return re.sub(r"\b\d{13,}\b", "", md)
|
| 896 |
-
|
| 897 |
-
|
| 898 |
-
def mask_debit_card_auth_codes(md: str) -> str:
|
| 899 |
-
"""
|
| 900 |
-
Mask 5–7 digit auth reference numbers after AUT (e.g. 'AUT 123024 VISA').
|
| 901 |
-
Extraction models often mistake those digits for currency.
|
| 902 |
-
"""
|
| 903 |
-
if not md:
|
| 904 |
-
return md
|
| 905 |
-
return re.sub(
|
| 906 |
-
r"(?i)\bAUT[\s,]+(\d{5,7})(?=\s+(?:VISA|DDA)\b)",
|
| 907 |
-
"AUT ******",
|
| 908 |
-
md,
|
| 909 |
-
)
|
| 910 |
-
|
| 911 |
-
|
| 912 |
-
def strip_glued_card_number_suffixes(md: str) -> str:
|
| 913 |
-
"""
|
| 914 |
-
Remove PAN-like digit runs glued after state/region markers (e.g. '*CT4085404035422892').
|
| 915 |
-
"""
|
| 916 |
-
if not md:
|
| 917 |
-
return md
|
| 918 |
-
return re.sub(r"(?i)(?<=[A-Za-z])(\d{13,})(?=</td>|<br\s*/?>)", "", md)
|
| 919 |
-
|
| 920 |
-
|
| 921 |
-
def mask_reference_numeric_ids(md: str) -> str:
|
| 922 |
-
"""
|
| 923 |
-
Mask long identifier-like numeric runs (ID/REF/TRN/TRACE/CARD/ACCT) so
|
| 924 |
-
extraction models don't misread them as transaction amounts.
|
| 925 |
-
"""
|
| 926 |
-
if not md:
|
| 927 |
-
return md
|
| 928 |
-
patterns = [
|
| 929 |
-
r"(?i)\b((?:orig\s+id|id|ind\s*id|co\s*id|ref|trace|trn|card|acct|account)\s*[:#]?\s*)(\d{7,})\b",
|
| 930 |
-
r"(?i)\b(text-\s*i?d\s*[:#]?\s*)(\d{7,})\b",
|
| 931 |
-
]
|
| 932 |
-
out = md
|
| 933 |
-
for pat in patterns:
|
| 934 |
-
out = re.sub(pat, lambda m: f"{m.group(1)}XXXXXXXX", out)
|
| 935 |
-
return out
|
| 936 |
-
|
| 937 |
-
|
| 938 |
-
def prune_td_statement_table_artifacts(md: str) -> str:
|
| 939 |
-
"""
|
| 940 |
-
TD-style statements often include:
|
| 941 |
-
- A 'Checks Paid' grid where check numbers land in the description column
|
| 942 |
-
- Subtotal rows where the posting date cell is blank but 'Subtotal:' is in description
|
| 943 |
-
|
| 944 |
-
Those rows are not normal ledger lines; if they survive into extraction they
|
| 945 |
-
double-count against Electronic Deposits / Payments totals and break reconciliation.
|
| 946 |
-
"""
|
| 947 |
-
if not md or "<table" not in md.lower():
|
| 948 |
-
return md
|
| 949 |
-
|
| 950 |
-
def _strip_trs(full_table: str) -> str:
|
| 951 |
-
plain = _cell_plain_text(full_table).lower()
|
| 952 |
-
is_checks = ("checks paid" in plain) and (
|
| 953 |
-
"serial no" in plain or "serial no." in plain or "checks:" in plain
|
| 954 |
-
)
|
| 955 |
-
is_posting_amt = ("posting date" in plain) and ("amount" in plain)
|
| 956 |
-
|
| 957 |
-
def maybe_drop_tr(tr_html: str) -> Optional[str]:
|
| 958 |
-
inner_m = re.search(r"<tr\b[^>]*>(.*)</tr>", tr_html, flags=re.IGNORECASE | re.DOTALL)
|
| 959 |
-
if not inner_m:
|
| 960 |
-
return tr_html
|
| 961 |
-
inner = inner_m.group(1)
|
| 962 |
-
cells = [_cell_plain_text(m.group(0)) for m in _CELL.finditer(inner)]
|
| 963 |
-
if not cells:
|
| 964 |
-
return tr_html
|
| 965 |
-
|
| 966 |
-
def _any_cell_subtotal(cs: List[str]) -> bool:
|
| 967 |
-
for c in cs:
|
| 968 |
-
cl = (c or "").strip().lower()
|
| 969 |
-
if cl.startswith("subtotal") or cl == "subtotal:":
|
| 970 |
-
return True
|
| 971 |
-
return False
|
| 972 |
-
|
| 973 |
-
if is_posting_amt and len(cells) >= 3:
|
| 974 |
-
dt = (cells[0] or "").strip()
|
| 975 |
-
desc = (cells[1] or "").strip()
|
| 976 |
-
desc_l = desc.lower()
|
| 977 |
-
dt_l = dt.lower()
|
| 978 |
-
amt_txt = (cells[-1] or "").strip()
|
| 979 |
-
amt = _parse_amount_or_none(amt_txt)
|
| 980 |
-
if _any_cell_subtotal(cells):
|
| 981 |
-
return ""
|
| 982 |
-
# Section headers / rolled-up lines (not individual postings).
|
| 983 |
-
if re.match(r"(?i)^(electronic\s+deposits|deposits|other\s+credits|checks\s+paid|electronic\s+payments|other\s+withdrawals|service\s+charges)\s*$", dt):
|
| 984 |
-
return ""
|
| 985 |
-
if re.match(r"(?i)^subtotal", desc_l) or desc_l.startswith("subtotal"):
|
| 986 |
-
return ""
|
| 987 |
-
# Drop OCR subtotal/total lines that are not dated posting rows.
|
| 988 |
-
if (not _is_date_like(dt)) and any(
|
| 989 |
-
k in desc_l for k in ("subtotal", "total for this cycle", "total year to date")
|
| 990 |
-
):
|
| 991 |
-
return ""
|
| 992 |
-
# Drop rare glue rows where date is present but description is empty and amount is huge.
|
| 993 |
-
if _is_date_like(dt) and (not desc) and amt is not None and amt >= 50_000:
|
| 994 |
-
return ""
|
| 995 |
-
# OCR sometimes shifts a section subtotal into an amount column as if it were a deposit.
|
| 996 |
-
if _is_date_like(dt) and (not desc) and amt is not None and amt >= 20_000:
|
| 997 |
-
return ""
|
| 998 |
-
# Check numbers can land in DESCRIPTION; those are not spend lines.
|
| 999 |
-
if _is_date_like(dt) and _looks_like_check_serial_token(desc) and amt is not None and amt >= 500:
|
| 1000 |
-
return ""
|
| 1001 |
-
|
| 1002 |
-
if is_checks:
|
| 1003 |
-
# Drop printed check lines (DATE + SERIAL + AMOUNT), including 2-up rows.
|
| 1004 |
-
# Keep header rows like "SERIAL NO." (no date in col0) and keep subtotal rows.
|
| 1005 |
-
hit = False
|
| 1006 |
-
for i in range(0, max(0, len(cells) - 2)):
|
| 1007 |
-
if _is_date_like((cells[i] or "").strip()) and _looks_like_check_serial_token(cells[i + 1] or ""):
|
| 1008 |
-
hit = True
|
| 1009 |
-
break
|
| 1010 |
-
if hit:
|
| 1011 |
-
return ""
|
| 1012 |
-
|
| 1013 |
-
return tr_html
|
| 1014 |
-
|
| 1015 |
-
out_parts = []
|
| 1016 |
-
pos = 0
|
| 1017 |
-
for m in re.finditer(r"<tr\b[^>]*>.*?</tr>", full_table, flags=re.IGNORECASE | re.DOTALL):
|
| 1018 |
-
out_parts.append(full_table[pos : m.start()])
|
| 1019 |
-
repl = maybe_drop_tr(m.group(0))
|
| 1020 |
-
if repl is not None:
|
| 1021 |
-
out_parts.append(repl)
|
| 1022 |
-
pos = m.end()
|
| 1023 |
-
out_parts.append(full_table[pos:])
|
| 1024 |
-
return "".join(out_parts)
|
| 1025 |
-
|
| 1026 |
-
def _repl_table(m: re.Match) -> str:
|
| 1027 |
-
tbl = m.group(0)
|
| 1028 |
-
return _strip_trs(tbl)
|
| 1029 |
-
|
| 1030 |
-
return re.sub(r"<table\b[^>]*>.*?</table>", _repl_table, md, flags=re.IGNORECASE | re.DOTALL)
|
| 1031 |
-
|
| 1032 |
-
|
| 1033 |
-
def infer_credit_debit_from_balance_deltas(md: str) -> str:
|
| 1034 |
-
"""
|
| 1035 |
-
Lightweight reconciliation assist:
|
| 1036 |
-
- normalize a clean balance snapshot section
|
| 1037 |
-
- synthesize running balances for date/description/amount tables that lack balance
|
| 1038 |
-
"""
|
| 1039 |
-
if not md or "<table" not in md.lower():
|
| 1040 |
-
return md
|
| 1041 |
-
out = prune_td_statement_table_artifacts(md)
|
| 1042 |
-
out = normalize_balance_snapshot(out)
|
| 1043 |
-
if _synth_running_balance_enabled():
|
| 1044 |
-
out = synthesize_running_balances(out)
|
| 1045 |
-
return out
|
| 1046 |
-
|
| 1047 |
-
|
| 1048 |
-
def normalize_balance_snapshot(md: str) -> str:
|
| 1049 |
-
"""Replace noisy/incorrect snapshot text with values parsed from statement summary."""
|
| 1050 |
-
if not md:
|
| 1051 |
-
return md
|
| 1052 |
-
start_bal, end_bal = _parse_statement_edge_balances(md)
|
| 1053 |
-
if start_bal is None and end_bal is None:
|
| 1054 |
-
return md
|
| 1055 |
-
# Remove any existing markdown snapshot block at document start.
|
| 1056 |
-
md2 = re.sub(
|
| 1057 |
-
r"^\s*##\s*Balance\s+Snapshot\s*\n(?:[^\n]*\n){0,8}\s*",
|
| 1058 |
-
"",
|
| 1059 |
-
md,
|
| 1060 |
-
count=1,
|
| 1061 |
-
flags=re.IGNORECASE,
|
| 1062 |
-
)
|
| 1063 |
-
lines = ["## Balance Snapshot"]
|
| 1064 |
-
if start_bal is not None:
|
| 1065 |
-
lines.append(f"Beginning balance: {start_bal:,.2f}")
|
| 1066 |
-
if end_bal is not None:
|
| 1067 |
-
lines.append(f"Ending balance: {end_bal:,.2f}")
|
| 1068 |
-
return "\n".join(lines) + "\n\n" + md2.lstrip()
|
| 1069 |
-
|
| 1070 |
-
|
| 1071 |
-
def _parse_statement_edge_balances(md: str) -> Tuple[Optional[float], Optional[float]]:
|
| 1072 |
-
"""Extract statement beginning/ending balances using generic wording patterns."""
|
| 1073 |
-
# Prefer account-summary table rows (most reliable for statements).
|
| 1074 |
-
for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
|
| 1075 |
-
tbl = tm.group(0)
|
| 1076 |
-
plain_tbl = _cell_plain_text(tbl).lower()
|
| 1077 |
-
if "account summary" not in plain_tbl:
|
| 1078 |
-
continue
|
| 1079 |
-
start = None
|
| 1080 |
-
end = None
|
| 1081 |
-
rows = _extract_rows_plain_from_table(tbl)
|
| 1082 |
-
for r in rows:
|
| 1083 |
-
if not r:
|
| 1084 |
-
continue
|
| 1085 |
-
key = " ".join((c or "").strip().lower() for c in r[:2])
|
| 1086 |
-
amts = []
|
| 1087 |
-
for c in r:
|
| 1088 |
-
for m in re.finditer(r"-?\d{1,3}(?:,\d{3})*(?:\.\d{2})", c or ""):
|
| 1089 |
-
v = _parse_amount_or_none(m.group(0))
|
| 1090 |
-
if v is not None:
|
| 1091 |
-
amts.append(v)
|
| 1092 |
-
if not amts:
|
| 1093 |
-
continue
|
| 1094 |
-
if start is None and ("beginning balance" in key or "starting balance" in key):
|
| 1095 |
-
start = amts[0]
|
| 1096 |
-
if end is None and "ending balance" in key:
|
| 1097 |
-
end = amts[-1]
|
| 1098 |
-
if start is not None or end is not None:
|
| 1099 |
-
return start, end
|
| 1100 |
-
|
| 1101 |
-
# Fallback: free-text scan with all candidates, choose the largest magnitude match.
|
| 1102 |
-
# Additional fallback forms common in statements.
|
| 1103 |
-
start_cands = []
|
| 1104 |
-
for m in re.finditer(
|
| 1105 |
-
r"(?:Balance\s+Forward(?:\s+From)?|Beginning\s+balance\s+on)\s*[^\n$]{0,60}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)\+?",
|
| 1106 |
-
md,
|
| 1107 |
-
flags=re.IGNORECASE,
|
| 1108 |
-
):
|
| 1109 |
-
v = _parse_amount_or_none(m.group(1))
|
| 1110 |
-
if v is not None:
|
| 1111 |
-
start_cands.append(v)
|
| 1112 |
-
for m in re.finditer(
|
| 1113 |
-
r"(?:Beginning|Starting)\s+balance[^$\n]{0,120}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)",
|
| 1114 |
-
md,
|
| 1115 |
-
flags=re.IGNORECASE,
|
| 1116 |
-
):
|
| 1117 |
-
v = _parse_amount_or_none(m.group(1))
|
| 1118 |
-
if v is not None:
|
| 1119 |
-
start_cands.append(v)
|
| 1120 |
-
end_cands = []
|
| 1121 |
-
for m in re.finditer(
|
| 1122 |
-
r"Ending\s+balance(?:\s+on)?[^\n$]{0,60}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)\+?",
|
| 1123 |
-
md,
|
| 1124 |
-
flags=re.IGNORECASE,
|
| 1125 |
-
):
|
| 1126 |
-
v = _parse_amount_or_none(m.group(1))
|
| 1127 |
-
if v is not None:
|
| 1128 |
-
end_cands.append(v)
|
| 1129 |
-
for m in re.finditer(
|
| 1130 |
-
r"Ending\s+balance[^$\n]{0,120}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)",
|
| 1131 |
-
md,
|
| 1132 |
-
flags=re.IGNORECASE,
|
| 1133 |
-
):
|
| 1134 |
-
v = _parse_amount_or_none(m.group(1))
|
| 1135 |
-
if v is not None:
|
| 1136 |
-
end_cands.append(v)
|
| 1137 |
-
start = max(start_cands, key=lambda x: abs(x)) if start_cands else None
|
| 1138 |
-
end = max(end_cands, key=lambda x: abs(x)) if end_cands else None
|
| 1139 |
-
return start, end
|
| 1140 |
-
|
| 1141 |
-
|
| 1142 |
-
def _table_sign_bias(ctx: str) -> int:
|
| 1143 |
-
"""Estimate sign direction for amount-only tables from local context."""
|
| 1144 |
-
c = (ctx or "").lower()
|
| 1145 |
-
if re.search(r"\b(deposit|deposits|credit|credits|other credits|rtp\s*rcvd|money\s+in)\b", c):
|
| 1146 |
-
return 1
|
| 1147 |
-
if re.search(
|
| 1148 |
-
r"\b(payment|payments|withdrawal|withdrawals|debit|debits|checks?\s+paid|service\s+charge|fee|fees|money\s+out)\b",
|
| 1149 |
-
c,
|
| 1150 |
-
):
|
| 1151 |
-
return -1
|
| 1152 |
-
return 0
|
| 1153 |
-
|
| 1154 |
-
|
| 1155 |
-
def _signed_amount_from_row(desc: str, amount: Optional[float], table_bias: int) -> Optional[float]:
|
| 1156 |
-
if amount is None:
|
| 1157 |
-
return None
|
| 1158 |
-
d = (desc or "").lower()
|
| 1159 |
-
if re.search(r"\b(deposit|credit|recd|received|refund|interest|rtp\s*rcvd)\b", d):
|
| 1160 |
-
return abs(amount)
|
| 1161 |
-
if re.search(r"\b(payment|withdraw|debit|purchase|fee|charge|check|ach)\b", d):
|
| 1162 |
-
return -abs(amount)
|
| 1163 |
-
if table_bias > 0:
|
| 1164 |
-
return abs(amount)
|
| 1165 |
-
if table_bias < 0:
|
| 1166 |
-
return -abs(amount)
|
| 1167 |
-
return None
|
| 1168 |
-
|
| 1169 |
-
|
| 1170 |
-
def synthesize_running_balances(md: str) -> str:
|
| 1171 |
-
"""
|
| 1172 |
-
For transaction-like tables lacking a Balance column, synthesize running balances
|
| 1173 |
-
from statement beginning balance and signed amounts. Generic, pattern-based only.
|
| 1174 |
-
"""
|
| 1175 |
-
if not md or "<table" not in md.lower():
|
| 1176 |
-
return md
|
| 1177 |
-
start_bal, end_bal = _parse_statement_edge_balances(md)
|
| 1178 |
-
if start_bal is None:
|
| 1179 |
-
return md
|
| 1180 |
-
|
| 1181 |
-
table_re = re.compile(r"<table\b[^>]*>.*?</table>", re.IGNORECASE | re.DOTALL)
|
| 1182 |
-
table_matches = list(table_re.finditer(md))
|
| 1183 |
-
if not table_matches:
|
| 1184 |
-
return md
|
| 1185 |
-
|
| 1186 |
-
plans = []
|
| 1187 |
-
signed_known = []
|
| 1188 |
-
signed_unknown = []
|
| 1189 |
-
for ti, tm in enumerate(table_matches):
|
| 1190 |
-
full = tm.group(0)
|
| 1191 |
-
rows = _extract_rows_plain_from_table(full)
|
| 1192 |
-
if len(rows) < 3:
|
| 1193 |
-
continue
|
| 1194 |
-
hdr = [c.strip() for c in rows[0]]
|
| 1195 |
-
hdr_l = [h.lower() for h in hdr]
|
| 1196 |
-
date_i = next((i for i, h in enumerate(hdr_l) if "date" in h), None)
|
| 1197 |
-
desc_i = next((i for i, h in enumerate(hdr_l) if "description" in h or "memo" in h or "details" in h), None)
|
| 1198 |
-
if date_i is None or desc_i is None:
|
| 1199 |
-
continue
|
| 1200 |
-
balance_i = next((i for i, h in enumerate(hdr_l) if "balance" in h), None)
|
| 1201 |
-
amount_i = next((i for i, h in enumerate(hdr_l) if "amount" in h and "balance" not in h), None)
|
| 1202 |
-
credit_i = next((i for i, h in enumerate(hdr_l) if "credit" in h), None)
|
| 1203 |
-
debit_i = next((i for i, h in enumerate(hdr_l) if "debit" in h), None)
|
| 1204 |
-
if amount_i is None and (credit_i is None or debit_i is None):
|
| 1205 |
-
continue
|
| 1206 |
-
|
| 1207 |
-
ctx_left = re.sub(r"<[^>]+>", " ", md[max(0, tm.start() - 320): tm.start()])
|
| 1208 |
-
ctx_tbl = re.sub(r"<[^>]+>", " ", full[:800])
|
| 1209 |
-
bias = _table_sign_bias(ctx_left + " " + ctx_tbl)
|
| 1210 |
-
|
| 1211 |
-
max_len = max(len(hdr), max((len(r) for r in rows), default=0))
|
| 1212 |
-
recs = []
|
| 1213 |
-
for ri, r in enumerate(rows[1:], start=1):
|
| 1214 |
-
row = (r + [""] * (max_len - len(r)))[:max_len]
|
| 1215 |
-
dt = (row[date_i] or "").strip()
|
| 1216 |
-
if not _is_date_like(dt):
|
| 1217 |
-
continue
|
| 1218 |
-
desc = (row[desc_i] or "").strip()
|
| 1219 |
-
bal = (
|
| 1220 |
-
_parse_amount_or_none((row[balance_i] or "").strip())
|
| 1221 |
-
if balance_i is not None and balance_i < len(row)
|
| 1222 |
-
else None
|
| 1223 |
-
)
|
| 1224 |
-
if amount_i is not None and amount_i < len(row):
|
| 1225 |
-
amt = _parse_amount_or_none((row[amount_i] or "").strip())
|
| 1226 |
-
signed = _signed_amount_from_row(desc, amt, bias)
|
| 1227 |
-
else:
|
| 1228 |
-
cr = _parse_amount_or_none((row[credit_i] or "").strip()) if credit_i < len(row) else None
|
| 1229 |
-
db = _parse_amount_or_none((row[debit_i] or "").strip()) if debit_i < len(row) else None
|
| 1230 |
-
signed = None
|
| 1231 |
-
if cr is not None and db is None:
|
| 1232 |
-
signed = abs(cr)
|
| 1233 |
-
elif db is not None and cr is None:
|
| 1234 |
-
signed = -abs(db)
|
| 1235 |
-
recs.append({"ri": ri, "row": row, "signed": signed, "balance": bal})
|
| 1236 |
-
if signed is None:
|
| 1237 |
-
signed_unknown.append((ti, ri))
|
| 1238 |
-
else:
|
| 1239 |
-
signed_known.append(float(signed))
|
| 1240 |
-
if len(recs) >= 3:
|
| 1241 |
-
plans.append({"ti": ti, "hdr": hdr, "balance_i": balance_i, "records": recs})
|
| 1242 |
-
|
| 1243 |
-
if not plans:
|
| 1244 |
-
return md
|
| 1245 |
-
|
| 1246 |
-
# If signs are mostly inverted for this statement, flip unknown-bias outcomes globally.
|
| 1247 |
-
flip_unknown = False
|
| 1248 |
-
if end_bal is not None and signed_known:
|
| 1249 |
-
fwd = start_bal + sum(signed_known)
|
| 1250 |
-
rev = start_bal - sum(signed_known)
|
| 1251 |
-
flip_unknown = abs(rev - end_bal) + 1e-6 < abs(fwd - end_bal)
|
| 1252 |
-
|
| 1253 |
-
pieces = []
|
| 1254 |
-
last = 0
|
| 1255 |
-
for ti, tm in enumerate(table_matches):
|
| 1256 |
-
pieces.append(md[last:tm.start()])
|
| 1257 |
-
full = tm.group(0)
|
| 1258 |
-
plan = next((p for p in plans if p["ti"] == ti), None)
|
| 1259 |
-
if not plan:
|
| 1260 |
-
pieces.append(full)
|
| 1261 |
-
last = tm.end()
|
| 1262 |
-
continue
|
| 1263 |
-
|
| 1264 |
-
hdr = plan["hdr"][:]
|
| 1265 |
-
balance_i = plan["balance_i"]
|
| 1266 |
-
if balance_i is None:
|
| 1267 |
-
hdr.append("Balance")
|
| 1268 |
-
balance_i = len(hdr) - 1
|
| 1269 |
-
|
| 1270 |
-
run = start_bal
|
| 1271 |
-
row_by_ri = {}
|
| 1272 |
-
for rec in plan["records"]:
|
| 1273 |
-
signed = rec["signed"]
|
| 1274 |
-
if signed is None:
|
| 1275 |
-
row_by_ri[rec["ri"]] = rec["row"]
|
| 1276 |
-
continue
|
| 1277 |
-
if flip_unknown:
|
| 1278 |
-
signed = -signed
|
| 1279 |
-
run += signed
|
| 1280 |
-
row = rec["row"][:]
|
| 1281 |
-
if len(row) < len(hdr):
|
| 1282 |
-
row += [""] * (len(hdr) - len(row))
|
| 1283 |
-
row[balance_i] = _fmt_money(run)
|
| 1284 |
-
row_by_ri[rec["ri"]] = row
|
| 1285 |
-
|
| 1286 |
-
base_rows = _extract_rows_plain_from_table(full)
|
| 1287 |
-
if len(base_rows) < 2:
|
| 1288 |
-
pieces.append(full)
|
| 1289 |
-
last = tm.end()
|
| 1290 |
-
continue
|
| 1291 |
-
out_rows = ["<tr>" + "".join(f"<th>{html.escape(c)}</th>" for c in hdr) + "</tr>"]
|
| 1292 |
-
for ri, old in enumerate(base_rows[1:], start=1):
|
| 1293 |
-
row = row_by_ri.get(ri, old)
|
| 1294 |
-
row = (row + [""] * (len(hdr) - len(row)))[: len(hdr)]
|
| 1295 |
-
out_rows.append("<tr>" + "".join(f"<td>{html.escape((c or '').strip())}</td>" for c in row) + "</tr>")
|
| 1296 |
-
pieces.append("<table>\n" + "\n".join(out_rows) + "\n</table>")
|
| 1297 |
-
last = tm.end()
|
| 1298 |
-
|
| 1299 |
-
pieces.append(md[last:])
|
| 1300 |
-
return "".join(pieces)
|
| 1301 |
-
|
| 1302 |
-
|
| 1303 |
-
def _extract_daily_balance_by_date(md: str):
|
| 1304 |
-
"""
|
| 1305 |
-
Build a date->balance map from daily-balance style tables.
|
| 1306 |
-
Supports compact statements with repeated Date/Balance column pairs.
|
| 1307 |
-
"""
|
| 1308 |
-
out = {}
|
| 1309 |
-
for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
|
| 1310 |
-
t = tm.group(0)
|
| 1311 |
-
rows = _extract_rows_plain_from_table(t)
|
| 1312 |
-
if len(rows) < 2:
|
| 1313 |
-
continue
|
| 1314 |
-
header_idx = None
|
| 1315 |
-
date_cols = []
|
| 1316 |
-
bal_cols = []
|
| 1317 |
-
plain_t = _cell_plain_text(t).lower()
|
| 1318 |
-
prefer_amount_as_balance = "daily ending balance" in plain_t or "daily balance" in plain_t
|
| 1319 |
-
for hi, row in enumerate(rows):
|
| 1320 |
-
hdr = [c.strip().lower() for c in row]
|
| 1321 |
-
if not hdr:
|
| 1322 |
-
continue
|
| 1323 |
-
dcols = [i for i, c in enumerate(hdr) if "date" in c]
|
| 1324 |
-
bcols = [i for i, c in enumerate(hdr) if "balance" in c]
|
| 1325 |
-
amount_cols = [i for i, c in enumerate(hdr) if "amount" in c]
|
| 1326 |
-
if (prefer_amount_as_balance or (len(dcols) >= 2 and len(amount_cols) >= 2)) and not bcols:
|
| 1327 |
-
bcols = [i for i, c in enumerate(hdr) if "amount" in c]
|
| 1328 |
-
if dcols and bcols:
|
| 1329 |
-
header_idx = hi
|
| 1330 |
-
date_cols = dcols
|
| 1331 |
-
bal_cols = bcols
|
| 1332 |
-
break
|
| 1333 |
-
if header_idx is None:
|
| 1334 |
-
continue
|
| 1335 |
-
if not date_cols or not bal_cols:
|
| 1336 |
-
continue
|
| 1337 |
-
pairs = []
|
| 1338 |
-
for di in date_cols:
|
| 1339 |
-
bi = next((b for b in bal_cols if b > di), None)
|
| 1340 |
-
if bi is not None:
|
| 1341 |
-
pairs.append((di, bi))
|
| 1342 |
-
if not pairs:
|
| 1343 |
-
continue
|
| 1344 |
-
for r in rows[header_idx + 1 :]:
|
| 1345 |
-
for di, bi in pairs:
|
| 1346 |
-
if di >= len(r) or bi >= len(r):
|
| 1347 |
-
continue
|
| 1348 |
-
d = (r[di] or "").strip()
|
| 1349 |
-
if not re.search(r"\b\d{1,2}/\d{1,2}(?:/\d{2,4})?\b", d):
|
| 1350 |
-
continue
|
| 1351 |
-
m = re.search(r"\d{1,2}/\d{1,2}(?:/\d{2,4})?", d)
|
| 1352 |
-
if not m:
|
| 1353 |
-
continue
|
| 1354 |
-
key = m.group(0)
|
| 1355 |
-
bal = _parse_amount_or_none(r[bi])
|
| 1356 |
-
if bal is None:
|
| 1357 |
-
continue
|
| 1358 |
-
out[key] = bal
|
| 1359 |
-
short = "/".join(key.split("/")[:2])
|
| 1360 |
-
out[short] = bal
|
| 1361 |
-
return out
|
| 1362 |
-
|
| 1363 |
-
|
| 1364 |
-
def _parse_summary_components(md: str) -> List[Tuple[str, float]]:
|
| 1365 |
-
"""
|
| 1366 |
-
Parse account/checking summary category totals as signed components.
|
| 1367 |
-
"""
|
| 1368 |
-
comps: List[Tuple[str, float]] = []
|
| 1369 |
-
for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
|
| 1370 |
-
t = tm.group(0)
|
| 1371 |
-
plain = _cell_plain_text(t).lower()
|
| 1372 |
-
if ("account summary" not in plain) and ("checking summary" not in plain):
|
| 1373 |
-
continue
|
| 1374 |
-
rows = _extract_rows_plain_from_table(t)
|
| 1375 |
-
for r in rows:
|
| 1376 |
-
if not r:
|
| 1377 |
-
continue
|
| 1378 |
-
label = (r[0] or "").strip().lower()
|
| 1379 |
-
if not label:
|
| 1380 |
-
continue
|
| 1381 |
-
if "average" in label:
|
| 1382 |
-
continue
|
| 1383 |
-
if "beginning balance" in label or "ending balance" in label:
|
| 1384 |
-
continue
|
| 1385 |
-
amts = []
|
| 1386 |
-
# Prefer the first amount cell after label; summary tables often have
|
| 1387 |
-
# an informational trailing column that should not be treated as amount.
|
| 1388 |
-
for c in r[1:] if len(r) > 1 else r:
|
| 1389 |
-
for m in re.finditer(r"-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})", c or ""):
|
| 1390 |
-
v = _parse_amount_or_none(m.group(0))
|
| 1391 |
-
if v is not None:
|
| 1392 |
-
amts.append(v)
|
| 1393 |
-
if not amts:
|
| 1394 |
-
continue
|
| 1395 |
-
v = float(amts[0])
|
| 1396 |
-
if re.search(r"\b(deposit|credit|addition)\b", label):
|
| 1397 |
-
signed = abs(v)
|
| 1398 |
-
elif re.search(r"\b(withdraw|debit|check|fee|service charge)\b", label):
|
| 1399 |
-
signed = -abs(v)
|
| 1400 |
-
else:
|
| 1401 |
-
signed = v
|
| 1402 |
-
comps.append((label[:64], signed))
|
| 1403 |
-
if comps:
|
| 1404 |
-
break
|
| 1405 |
-
return comps
|
| 1406 |
-
|
| 1407 |
-
|
| 1408 |
-
def _force_summary_ledger_fallback(md: str) -> str:
|
| 1409 |
-
"""
|
| 1410 |
-
Fallback when daily balances are unavailable: synthesize a compact ledger
|
| 1411 |
-
from account-summary totals so statement reconciliation can still close.
|
| 1412 |
-
"""
|
| 1413 |
-
if not md or "<table" not in md.lower():
|
| 1414 |
-
return md
|
| 1415 |
-
start_bal, end_bal = _parse_statement_edge_balances(md)
|
| 1416 |
-
if start_bal is None or end_bal is None:
|
| 1417 |
-
return md
|
| 1418 |
-
comps = _parse_summary_components(md)
|
| 1419 |
-
if not comps:
|
| 1420 |
-
return md
|
| 1421 |
-
|
| 1422 |
-
expected = float(start_bal) + sum(v for _, v in comps)
|
| 1423 |
-
alt_expected = float(-start_bal) + sum(v for _, v in comps)
|
| 1424 |
-
# Some OCR paths lose the minus sign on beginning balances; recover it here.
|
| 1425 |
-
if abs(alt_expected - float(end_bal)) + 1e-6 < abs(expected - float(end_bal)):
|
| 1426 |
-
start_bal = -float(start_bal)
|
| 1427 |
-
expected = alt_expected
|
| 1428 |
-
if abs(expected - float(end_bal)) > 0.05:
|
| 1429 |
-
return md
|
| 1430 |
-
|
| 1431 |
-
def _drop_noisy_table(m: re.Match) -> str:
|
| 1432 |
-
t = m.group(0)
|
| 1433 |
-
plain = _cell_plain_text(t).lower()
|
| 1434 |
-
if "account summary" in plain or "checking summary" in plain:
|
| 1435 |
-
return t
|
| 1436 |
-
if "daily ending balance" in plain or "daily balance" in plain:
|
| 1437 |
-
return t
|
| 1438 |
-
rows = _extract_rows_plain_from_table(t)
|
| 1439 |
-
hdr_rows = rows[:3] if rows else []
|
| 1440 |
-
txn_like = False
|
| 1441 |
-
for hr in hdr_rows:
|
| 1442 |
-
hl = " ".join(hr).lower()
|
| 1443 |
-
if "date" in hl and "amount" in hl and ("description" in hl or "memo" in hl):
|
| 1444 |
-
txn_like = True
|
| 1445 |
-
break
|
| 1446 |
-
if len(rows) >= 4 and txn_like:
|
| 1447 |
-
return ""
|
| 1448 |
-
return t
|
| 1449 |
-
|
| 1450 |
-
rows = ["<tr><th>Date</th><th>Description</th><th>Credit</th><th>Debit</th><th>Balance</th></tr>"]
|
| 1451 |
-
run = float(start_bal)
|
| 1452 |
-
rows.append(f"<tr><td>01/01</td><td>BALANCE FORWARD</td><td></td><td></td><td>{_fmt_money(run)}</td></tr>")
|
| 1453 |
-
day = 2
|
| 1454 |
-
for lbl, signed in comps:
|
| 1455 |
-
run += signed
|
| 1456 |
-
credit = _fmt_money(signed) if signed > 0 else ""
|
| 1457 |
-
debit = _fmt_money(abs(signed)) if signed < 0 else ""
|
| 1458 |
-
rows.append(f"<tr><td>01/{day:02d}</td><td>{html.escape(lbl.upper())}</td><td>{credit}</td><td>{debit}</td><td>{_fmt_money(run)}</td></tr>")
|
| 1459 |
-
day += 1
|
| 1460 |
-
synth = "<table>\n" + "\n".join(rows) + "\n</table>"
|
| 1461 |
-
cleaned = re.sub(r"<table\b[^>]*>.*?</table>", _drop_noisy_table, md, flags=re.IGNORECASE | re.DOTALL)
|
| 1462 |
-
preface = (
|
| 1463 |
-
"## Balance Snapshot\n"
|
| 1464 |
-
f"Beginning balance: {_fmt_money(float(start_bal))}\n"
|
| 1465 |
-
f"Ending balance: {_fmt_money(float(end_bal))}\n\n"
|
| 1466 |
-
"The following reconciliation ledger is a normalized summary derived from statement totals. "
|
| 1467 |
-
"It is intended to provide a stable machine-readable trail of signed amounts and running balances. "
|
| 1468 |
-
"Rows are ordered as ledger events and balances are carried forward deterministically.\n\n"
|
| 1469 |
-
)
|
| 1470 |
-
return preface + cleaned + "\n\n" + synth
|
| 1471 |
-
|
| 1472 |
-
|
| 1473 |
-
def _force_ledger_from_daily_balances(md: str) -> str:
|
| 1474 |
-
"""
|
| 1475 |
-
Build a deterministic ledger from daily balance summary and remove noisy posting tables.
|
| 1476 |
-
This gives downstream extraction a clean credit/debit stream that should reconcile exactly
|
| 1477 |
-
to beginning/ending balances when daily balances are reliable.
|
| 1478 |
-
"""
|
| 1479 |
-
if not md or "<table" not in md.lower():
|
| 1480 |
-
return md
|
| 1481 |
-
|
| 1482 |
-
by_date = _extract_daily_balance_by_date(md)
|
| 1483 |
-
if len(by_date) < 3:
|
| 1484 |
-
return md
|
| 1485 |
-
|
| 1486 |
-
points = []
|
| 1487 |
-
seen = set()
|
| 1488 |
-
for k, v in by_date.items():
|
| 1489 |
-
m = re.match(r"^\s*(\d{1,2})/(\d{1,2})(?:/\d{2,4})?\s*$", k or "")
|
| 1490 |
-
if not m:
|
| 1491 |
-
continue
|
| 1492 |
-
mm = int(m.group(1))
|
| 1493 |
-
dd = int(m.group(2))
|
| 1494 |
-
key = (mm, dd)
|
| 1495 |
-
if key in seen:
|
| 1496 |
-
continue
|
| 1497 |
-
seen.add(key)
|
| 1498 |
-
points.append((mm, dd, v))
|
| 1499 |
-
has_jan = any(mm == 1 for mm, _, _ in points)
|
| 1500 |
-
# Statements often include prior-month carry-forward like 12/31 followed by Jan activity.
|
| 1501 |
-
# Keep late-December carry-forward rows before January rows.
|
| 1502 |
-
points.sort(key=lambda x: ((-1 if has_jan and x[0] == 12 else x[0]), x[1]))
|
| 1503 |
-
if len(points) < 3:
|
| 1504 |
-
return md
|
| 1505 |
-
|
| 1506 |
-
first_bal = points[0][2]
|
| 1507 |
-
last_bal = points[-1][2]
|
| 1508 |
-
if first_bal is None or last_bal is None:
|
| 1509 |
-
return md
|
| 1510 |
-
|
| 1511 |
-
# Remove noisy posting tables but keep account summary and daily balance summary tables.
|
| 1512 |
-
def _drop_noisy_table(m: re.Match) -> str:
|
| 1513 |
-
t = m.group(0)
|
| 1514 |
-
plain = _cell_plain_text(t).lower()
|
| 1515 |
-
if "daily balance summary" in plain or "daily ending balance" in plain or "daily balance" in plain:
|
| 1516 |
-
return t
|
| 1517 |
-
if "account summary" in plain or "checking summary" in plain:
|
| 1518 |
-
return t
|
| 1519 |
-
return ""
|
| 1520 |
-
|
| 1521 |
-
rows = [
|
| 1522 |
-
"<tr><th>Date</th><th>Description</th><th>Credit</th><th>Debit</th><th>Balance</th></tr>",
|
| 1523 |
-
f"<tr><td>{points[0][0]:02d}/{points[0][1]:02d}</td><td>BALANCE FORWARD</td><td></td><td></td><td>{_fmt_money(first_bal)}</td></tr>",
|
| 1524 |
-
]
|
| 1525 |
-
for i in range(1, len(points)):
|
| 1526 |
-
mm, dd, bal = points[i]
|
| 1527 |
-
prev = points[i - 1][2]
|
| 1528 |
-
delta = float(bal) - float(prev)
|
| 1529 |
-
credit = _fmt_money(delta) if delta > 0 else ""
|
| 1530 |
-
debit = _fmt_money(abs(delta)) if delta < 0 else ""
|
| 1531 |
-
rows.append(
|
| 1532 |
-
f"<tr><td>{mm:02d}/{dd:02d}</td><td>NET DAILY CHANGE</td><td>{credit}</td><td>{debit}</td><td>{_fmt_money(bal)}</td></tr>"
|
| 1533 |
-
)
|
| 1534 |
-
synth = "<table>\n" + "\n".join(rows) + "\n</table>"
|
| 1535 |
-
cleaned = re.sub(r"<table\b[^>]*>.*?</table>", _drop_noisy_table, md, flags=re.IGNORECASE | re.DOTALL)
|
| 1536 |
-
preface = (
|
| 1537 |
-
"## Balance Snapshot\n"
|
| 1538 |
-
f"Beginning balance: {_fmt_money(float(first_bal))}\n"
|
| 1539 |
-
f"Ending balance: {_fmt_money(float(last_bal))}\n\n"
|
| 1540 |
-
"The following reconciliation ledger is synthesized from daily balance points. "
|
| 1541 |
-
"Each row is the net day-over-day movement with a deterministic running balance. "
|
| 1542 |
-
"This representation is designed for robust downstream extraction and reconciliation.\n\n"
|
| 1543 |
-
)
|
| 1544 |
-
return preface + cleaned + "\n\n" + synth
|
| 1545 |
-
|
| 1546 |
-
|
| 1547 |
def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
|
| 1548 |
import pymupdf as fitz
|
| 1549 |
from PIL import Image
|
|
@@ -1570,7 +221,6 @@ def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
|
|
| 1570 |
canvas.paste(img, (pad_l, pad_t))
|
| 1571 |
img = canvas
|
| 1572 |
|
| 1573 |
-
# Include a per-run unique tag to avoid filename collisions across parallel local runs.
|
| 1574 |
uniq = uuid.uuid4().hex[:10]
|
| 1575 |
img_path = os.path.join(tempfile.gettempdir(), f"glmocr_page_{os.getpid()}_{uniq}_{i}.png")
|
| 1576 |
img.save(img_path, "PNG", compress_level=PAGE_PNG_COMPRESS_LEVEL)
|
|
@@ -1644,10 +294,10 @@ def run_ocr(uploaded_file):
|
|
| 1644 |
if not (hdr and hdr.strip()) and page_num < len(page_images):
|
| 1645 |
hdr = ocr_zone(page_images[page_num], 0, he)
|
| 1646 |
if hdr and hdr.strip():
|
| 1647 |
-
parts.append(
|
| 1648 |
|
| 1649 |
if page_md and page_md.strip():
|
| 1650 |
-
parts.append(
|
| 1651 |
|
| 1652 |
if ENABLE_FOOTER_OCR and page_num < len(page_images):
|
| 1653 |
ftr = ""
|
|
@@ -1658,7 +308,7 @@ def run_ocr(uploaded_file):
|
|
| 1658 |
if not (ftr and ftr.strip()):
|
| 1659 |
ftr = ocr_zone(page_images[page_num], fs, 1.0)
|
| 1660 |
if ftr and ftr.strip():
|
| 1661 |
-
ftr_clean =
|
| 1662 |
|
| 1663 |
ftr_first_line = next(
|
| 1664 |
(ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
|
|
@@ -1681,20 +331,6 @@ def run_ocr(uploaded_file):
|
|
| 1681 |
all_pages.append("\n\n".join(parts))
|
| 1682 |
|
| 1683 |
merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
|
| 1684 |
-
if merged and merged != "(No content)" and not merged.lstrip().startswith("Error:"):
|
| 1685 |
-
merged = stabilize_table_markup(merged, rounds=2)
|
| 1686 |
-
merged = repair_thead_cell_semantics(merged)
|
| 1687 |
-
merged = infer_credit_debit_from_balance_deltas(merged)
|
| 1688 |
-
summary_forced = _force_summary_ledger_fallback(merged)
|
| 1689 |
-
if summary_forced != merged:
|
| 1690 |
-
merged = summary_forced
|
| 1691 |
-
else:
|
| 1692 |
-
merged = _force_ledger_from_daily_balances(merged)
|
| 1693 |
-
merged = strip_ultra_long_digit_tokens(merged)
|
| 1694 |
-
merged = mask_reference_numeric_ids(merged)
|
| 1695 |
-
merged = mask_debit_card_auth_codes(merged)
|
| 1696 |
-
merged = strip_glued_card_number_suffixes(merged)
|
| 1697 |
-
merged = strip_degenerate_html_tables(merged)
|
| 1698 |
return merged
|
| 1699 |
|
| 1700 |
except Exception as e:
|
|
@@ -1716,17 +352,13 @@ def _create_gradio_demo():
|
|
| 1716 |
import gradio as gr
|
| 1717 |
|
| 1718 |
with gr.Blocks(title="GLM-OCR (simple)") as demo:
|
| 1719 |
-
gr.Markdown(
|
| 1720 |
-
"# GLM-OCR (simple)\n"
|
| 1721 |
-
"Upload a PDF or image. Header and footer bands are included; "
|
| 1722 |
-
"body OCR is passed through with only light markdown cleanup."
|
| 1723 |
-
)
|
| 1724 |
file_in = gr.File(
|
| 1725 |
label="Upload PDF or image",
|
| 1726 |
file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"],
|
| 1727 |
)
|
| 1728 |
run_btn = gr.Button("Run OCR", variant="primary")
|
| 1729 |
-
out = gr.Textbox(lines=40, label="Output (markdown /
|
| 1730 |
run_btn.click(fn=run_ocr, inputs=file_in, outputs=out)
|
| 1731 |
return demo
|
| 1732 |
|
|
|
|
| 1 |
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import logging
|
| 4 |
import os
|
| 5 |
import re
|
| 6 |
import tempfile
|
| 7 |
import uuid
|
| 8 |
+
from typing import List, Tuple
|
|
|
|
| 9 |
|
| 10 |
import yaml
|
| 11 |
|
|
|
|
| 15 |
GLMOCR_BASE = os.path.dirname(glmocr.__file__)
|
| 16 |
CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
|
| 17 |
except ImportError:
|
| 18 |
+
glmocr = None
|
| 19 |
GLMOCR_BASE = ""
|
| 20 |
CONFIG_PATH = ""
|
| 21 |
|
| 22 |
log = logging.getLogger("glmocr_simple_app")
|
| 23 |
logging.basicConfig(level=logging.INFO)
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
GLMOCR_API_KEY = "cee1d52dd91a4ab591b3f6e105f8ad89.LgbQTECuzX0zrito"
|
| 26 |
if not GLMOCR_API_KEY:
|
| 27 |
log.warning(
|
| 28 |
"No ZHIPU_API_KEY or GLMOCR_API_KEY in environment; GlmOcr() will fail until you set one."
|
| 29 |
)
|
| 30 |
|
|
|
|
|
|
|
| 31 |
RENDER_SCALE = 3.0
|
|
|
|
|
|
|
|
|
|
| 32 |
PAD_LEFT_FRAC = 0.035
|
| 33 |
PAD_RIGHT_FRAC = 0.10
|
| 34 |
PAD_TOP_FRAC = 0.018
|
| 35 |
PAD_BOTTOM_FRAC = 0.018
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
ENABLE_CONTRAST = True
|
|
|
|
| 38 |
CONTRAST_FACTOR = 1.18
|
|
|
|
|
|
|
| 39 |
ENABLE_UNSHARP = True
|
| 40 |
UNSHARP_RADIUS = 0.78
|
| 41 |
UNSHARP_PERCENT = 76
|
|
|
|
| 43 |
|
| 44 |
DEFAULT_ZONE_FRAC = 0.12
|
| 45 |
PDF_HEADER_BAND_FRAC = 0.10
|
|
|
|
| 46 |
ENABLE_FOOTER_OCR = True
|
| 47 |
PDF_FOOTER_BAND_FRAC = 0.88
|
| 48 |
|
| 49 |
MIN_CROP_HEIGHT = 112
|
| 50 |
MIN_CROP_PIXELS = 112 * 112
|
|
|
|
|
|
|
| 51 |
PAGE_PNG_COMPRESS_LEVEL = 3
|
|
|
|
| 52 |
ZONE_JPEG_QUALITY = 95
|
| 53 |
|
|
|
|
|
|
|
| 54 |
_parser = None
|
| 55 |
|
| 56 |
|
| 57 |
def _enhance_raster_for_ocr(img):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
from PIL import ImageEnhance, ImageFilter
|
| 59 |
|
| 60 |
if ENABLE_CONTRAST:
|
|
|
|
| 195 |
return ""
|
| 196 |
|
| 197 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
|
| 199 |
import pymupdf as fitz
|
| 200 |
from PIL import Image
|
|
|
|
| 221 |
canvas.paste(img, (pad_l, pad_t))
|
| 222 |
img = canvas
|
| 223 |
|
|
|
|
| 224 |
uniq = uuid.uuid4().hex[:10]
|
| 225 |
img_path = os.path.join(tempfile.gettempdir(), f"glmocr_page_{os.getpid()}_{uniq}_{i}.png")
|
| 226 |
img.save(img_path, "PNG", compress_level=PAGE_PNG_COMPRESS_LEVEL)
|
|
|
|
| 294 |
if not (hdr and hdr.strip()) and page_num < len(page_images):
|
| 295 |
hdr = ocr_zone(page_images[page_num], 0, he)
|
| 296 |
if hdr and hdr.strip():
|
| 297 |
+
parts.append(hdr.strip())
|
| 298 |
|
| 299 |
if page_md and page_md.strip():
|
| 300 |
+
parts.append(page_md.strip())
|
| 301 |
|
| 302 |
if ENABLE_FOOTER_OCR and page_num < len(page_images):
|
| 303 |
ftr = ""
|
|
|
|
| 308 |
if not (ftr and ftr.strip()):
|
| 309 |
ftr = ocr_zone(page_images[page_num], fs, 1.0)
|
| 310 |
if ftr and ftr.strip():
|
| 311 |
+
ftr_clean = ftr.strip()
|
| 312 |
|
| 313 |
ftr_first_line = next(
|
| 314 |
(ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
|
|
|
|
| 331 |
all_pages.append("\n\n".join(parts))
|
| 332 |
|
| 333 |
merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 334 |
return merged
|
| 335 |
|
| 336 |
except Exception as e:
|
|
|
|
| 352 |
import gradio as gr
|
| 353 |
|
| 354 |
with gr.Blocks(title="GLM-OCR (simple)") as demo:
|
| 355 |
+
gr.Markdown("# GLM-OCR")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 356 |
file_in = gr.File(
|
| 357 |
label="Upload PDF or image",
|
| 358 |
file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"],
|
| 359 |
)
|
| 360 |
run_btn = gr.Button("Run OCR", variant="primary")
|
| 361 |
+
out = gr.Textbox(lines=40, label="Output (markdown / HTML from model)")
|
| 362 |
run_btn.click(fn=run_ocr, inputs=file_in, outputs=out)
|
| 363 |
return demo
|
| 364 |
|