rehan953 commited on
Commit
3be8674
·
verified ·
1 Parent(s): 5ad86be

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +7 -1375
app.py CHANGED
@@ -1,61 +1,11 @@
1
  #!/usr/bin/env python3
2
- """
3
- Simplified GLM-OCR Hugging Face / local Gradio app.
4
-
5
- Scope (intentionally small):
6
- - PDF → padded high-DPI page images → GLM-OCR body markdown
7
- - Header band: PDF text extraction first, optional header OCR fallback
8
- - Footer band: same pattern, with light dedup so we do not paste a full
9
- transaction dump twice when the body already captured it
10
-
11
- Universal image pipeline (same for every PDF, no keywords / no bank logic):
12
- - Higher rasterization scale + extra white padding so fine print, boxed
13
- section labels, and right-aligned amounts sit farther from the clip edge.
14
- - Mild contrast + unsharp mask on every raster sent to the model so
15
- thin rules and small glyphs are easier to read before recognition.
16
-
17
- Explicitly omitted vs the heavy Space build:
18
- - No text-layer row injection, institution-specific splits (UCB / Navy /
19
- TD / First Horizon / …), or doc-wide dedupe passes.
20
-
21
- Included (data-driven, no institution names):
22
- - HTML tables: modal logical width from rowspan-free rows (colspan-aware);
23
- pad short rows; trim trailing empty cells; expand each row to a logical
24
- grid, then slide a solitary amount token past trailing blank logical slots
25
- into the rightmost slot (colspan-aware). Rowspan rows are skipped for edits
26
- but do not disable an entire table. Stabilization runs in multiple passes.
27
- - Split single cells that clearly contain transaction amount + trailing balance
28
- (two money tokens, tight gap) into two cells so classifiers can see a balance column.
29
- - thead uses th only; degenerate empty/sparse non-financial tables are dropped.
30
-
31
- Configure GLMOCR_API_KEY, GLM_OCR_API_KEY, or ZHIPU_API_KEY (environment). Optional: glmocr + gradio +
32
- pymupdf + pillow installed.
33
- """
34
-
35
- # Patch asyncio first (before Gradio imports it) to reduce Python 3.13 loop noise
36
- import asyncio
37
 
38
- try:
39
- _orig_close = asyncio.BaseEventLoop.close
40
-
41
- def _safe_close(self):
42
- try:
43
- _orig_close(self)
44
- except (ValueError, OSError):
45
- pass
46
-
47
- asyncio.BaseEventLoop.close = _safe_close
48
- except Exception:
49
- pass
50
-
51
- import html
52
  import logging
53
  import os
54
  import re
55
  import tempfile
56
  import uuid
57
- from collections import Counter
58
- from typing import List, Optional, Tuple
59
 
60
  import yaml
61
 
@@ -65,46 +15,27 @@ try:
65
  GLMOCR_BASE = os.path.dirname(glmocr.__file__)
66
  CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
67
  except ImportError:
68
- glmocr = None # type: ignore
69
  GLMOCR_BASE = ""
70
  CONFIG_PATH = ""
71
 
72
  log = logging.getLogger("glmocr_simple_app")
73
  logging.basicConfig(level=logging.INFO)
74
 
75
- # ---------------------------------------------------------------------------
76
- # Settings — tuned for dense financial PDFs; applies to every document
77
- # ---------------------------------------------------------------------------
78
-
79
- # Never commit secrets: Space / local runs use ZHIPU_API_KEY or GLMOCR_API_KEY.
80
  GLMOCR_API_KEY = "cee1d52dd91a4ab591b3f6e105f8ad89.LgbQTECuzX0zrito"
81
  if not GLMOCR_API_KEY:
82
  log.warning(
83
  "No ZHIPU_API_KEY or GLMOCR_API_KEY in environment; GlmOcr() will fail until you set one."
84
  )
85
 
86
- # Rasterization: higher scale = more pixels per PDF point (helps small type,
87
- # boxed headers, and narrow columns). Same constant for all uploads.
88
  RENDER_SCALE = 3.0
89
-
90
- # White margin as a fraction of page width/height after render. Extra right
91
- # margin helps right-aligned currency columns that hug the page edge.
92
  PAD_LEFT_FRAC = 0.035
93
  PAD_RIGHT_FRAC = 0.10
94
  PAD_TOP_FRAC = 0.018
95
  PAD_BOTTOM_FRAC = 0.018
96
 
97
- # Synthesized running-balance columns help some heuristics but downstream LLM
98
- # extractors may mis-read them as credits; default off (set GLMOCR_SYNTH_RUNNING_BALANCE=1 to enable).
99
- def _synth_running_balance_enabled() -> bool:
100
- return os.environ.get("GLMOCR_SYNTH_RUNNING_BALANCE", "").lower() in ("1", "true", "yes")
101
-
102
-
103
  ENABLE_CONTRAST = True
104
- # Slight contrast lift only; same factor for every file.
105
  CONTRAST_FACTOR = 1.18
106
-
107
- # Subtle edge enhancement after contrast (helps hairlines and small digits).
108
  ENABLE_UNSHARP = True
109
  UNSHARP_RADIUS = 0.78
110
  UNSHARP_PERCENT = 76
@@ -112,29 +43,18 @@ UNSHARP_THRESHOLD = 1
112
 
113
  DEFAULT_ZONE_FRAC = 0.12
114
  PDF_HEADER_BAND_FRAC = 0.10
115
-
116
  ENABLE_FOOTER_OCR = True
117
  PDF_FOOTER_BAND_FRAC = 0.88
118
 
119
  MIN_CROP_HEIGHT = 112
120
  MIN_CROP_PIXELS = 112 * 112
121
-
122
- # PNG compression 0–9; lower = less loss before GLM-OCR (same for all PDFs).
123
  PAGE_PNG_COMPRESS_LEVEL = 3
124
- # JPEG quality for small header/footer crops sent to the API.
125
  ZONE_JPEG_QUALITY = 95
126
 
127
- MIN_PDF_TEXT_CHARS_NATIVE_LAYER = 1500
128
-
129
  _parser = None
130
 
131
 
132
  def _enhance_raster_for_ocr(img):
133
- """
134
- Improve legibility of every raster passed to GLM-OCR (full pages and
135
- header/footer crops). No document text or keywords — same pipeline for
136
- all PDFs and images.
137
- """
138
  from PIL import ImageEnhance, ImageFilter
139
 
140
  if ENABLE_CONTRAST:
@@ -275,1275 +195,6 @@ def ocr_zone(image_path, y_start_frac, y_end_frac):
275
  return ""
276
 
277
 
278
- def fix_account_number(hdr: str) -> str:
279
- if not hdr:
280
- return hdr
281
- if "Account Number:" in hdr and "Account Number: " not in hdr:
282
- m = re.search(r"[0-9]{5,}", hdr)
283
- if m:
284
- hdr = hdr.replace("Account Number:", "Account Number: " + m.group(0))
285
- acct_match = re.search(r"Account Number: ([0-9]{5,})", hdr)
286
- if acct_match:
287
- acct = acct_match.group(1)
288
- if hdr.startswith(acct):
289
- hdr = hdr[len(acct) :].lstrip()
290
- return hdr
291
-
292
-
293
- def close_unclosed_html(md: str) -> str:
294
- if not md:
295
- return md
296
- open_tags = re.findall(r"<(table|tbody|thead|tr|td|th)\b", md, flags=re.IGNORECASE)
297
- close_tags = re.findall(r"</(table|tbody|thead|tr|td|th)>", md, flags=re.IGNORECASE)
298
-
299
- def count(tags, name):
300
- return sum(1 for t in tags if t.lower() == name)
301
-
302
- for tag in reversed(["td", "th", "tr", "thead", "tbody", "table"]):
303
- opened = count(open_tags, tag)
304
- closed = count(close_tags, tag)
305
- if opened > closed:
306
- md += ("</%s>" % tag) * (opened - closed)
307
- return md
308
-
309
-
310
- _TR_OPEN = re.compile(r"<tr\b([^>]*)>", re.IGNORECASE)
311
- _TR_CLOSE = re.compile(r"</tr>", re.IGNORECASE)
312
- _CELL = re.compile(
313
- r"<(td|th)(\b[^>]*?)>((?:(?!</?(?:td|th)\b).)*?)</(td|th)\s*>",
314
- re.IGNORECASE | re.DOTALL,
315
- )
316
-
317
- # Currency tokens inside a cell (not anchored); used to split merged amount+balance.
318
- _MONEY_IN_TEXT = re.compile(
319
- r"(?:\$|€|£)?\s*-?\d{1,3}(?:,\d{3})*\.\d{2}\b|(?:\$|€|£)?\s*-?\d+\.\d{2}\b"
320
- )
321
- _EOL_MONEY = re.compile(
322
- r"(?:\$|€|£)?\s*-?\d{1,3}(?:,\d{3})*\.\d{2}$|(?:\$|€|£)?\s*-?\d+\.\d{2}$"
323
- )
324
-
325
-
326
- def _split_cell_trailing_balance(full_cell: str) -> List[str]:
327
- """
328
- When OCR puts transaction amount and running balance in one <td>, split into
329
- two cells so classifiers can assign separate columns. Uses only currency
330
- patterns and whitespace gaps (no header names).
331
- """
332
- plain = _cell_plain_text(full_cell)
333
- if len(plain) < 10:
334
- return [full_cell]
335
- spans = [(m.start(), m.end()) for m in _MONEY_IN_TEXT.finditer(plain)]
336
- if len(spans) < 2:
337
- return [full_cell]
338
- (a0, a1), (b0, b1) = spans[-2], spans[-1]
339
- if b1 < len(plain) - 16:
340
- return [full_cell]
341
- gap = plain[a1:b0]
342
- if re.search(r"[A-Za-z]{2,}", gap):
343
- return [full_cell]
344
- if len(gap) > 14:
345
- return [full_cell]
346
- left = plain[:b0].strip()
347
- right = plain[b0:].strip()
348
- if not left or not right:
349
- return [full_cell]
350
- m = _CELL.fullmatch(full_cell.strip())
351
- if not m:
352
- return [full_cell]
353
- tag, attrs = m.group(1), m.group(2)
354
- return [
355
- f"<{tag}{attrs}>{html.escape(left)}</{tag}>",
356
- f"<{tag}{attrs}>{html.escape(right)}</{tag}>",
357
- ]
358
-
359
-
360
- def _expand_tr_inner_split_merged(tr_inner: str) -> str:
361
- """Insert extra td/th where a single cell clearly holds amount + trailing balance."""
362
- entries = _cell_entries(tr_inner)
363
- if not entries:
364
- return tr_inner
365
- parts: List[str] = []
366
- for full, span in entries:
367
- if span != 1:
368
- parts.append(full)
369
- else:
370
- parts.extend(_split_cell_trailing_balance(full))
371
- return "".join(parts)
372
-
373
-
374
- def _cell_entries(tr_inner: str) -> List[Tuple[str, int]]:
375
- """(full_cell_html, logical_width) for each td/th; 0 cells if unparseable."""
376
- out: List[Tuple[str, int]] = []
377
- for m in _CELL.finditer(tr_inner):
378
- open_name, attrs, _body, close_name = m.group(1), m.group(2), m.group(3), m.group(4)
379
- if open_name.lower() != close_name.lower():
380
- continue
381
- cm = re.search(r"colspan\s*=\s*[\"']?(\d+)", attrs, flags=re.IGNORECASE)
382
- span = int(cm.group(1)) if cm else 1
383
- span = max(1, span)
384
- out.append((m.group(0), span))
385
- return out
386
-
387
-
388
- def _logical_row_width(entries: List[Tuple[str, int]]) -> int:
389
- return sum(s for _f, s in entries)
390
-
391
-
392
- def _cell_text_empty(full_cell: str) -> bool:
393
- m = _CELL.fullmatch(full_cell.strip())
394
- if not m:
395
- inner = re.sub(r"<[^>]+>", " ", full_cell)
396
- else:
397
- inner = m.group(3)
398
- inner = re.sub(r"\s+", " ", inner).strip()
399
- inner = html.unescape(inner)
400
- return inner == ""
401
-
402
-
403
- def _cell_plain_text(full_cell: str) -> str:
404
- """Visible text of one td/th, no tags."""
405
- m = _CELL.fullmatch(full_cell.strip())
406
- if not m:
407
- t = re.sub(r"<[^>]+>", " ", full_cell)
408
- else:
409
- t = m.group(3)
410
- t = html.unescape(re.sub(r"\s+", " ", t).strip())
411
- return t
412
-
413
-
414
- def _is_whole_cell_currency(text: str) -> bool:
415
- """
416
- True iff the cell is nothing but a currency-looking amount (optional $, commas, 2 decimals).
417
- Excludes dates (slashes) and arbitrary prose — not keyed to column headers.
418
- """
419
- t = (text or "").strip().strip("* \t\u00a0")
420
- if not t or "/" in t:
421
- return False
422
- return bool(
423
- re.fullmatch(
424
- r"-?(?:\$|€|£)?\s*\d{1,3}(?:,\d{3})*\.\d{2}\s*",
425
- t,
426
- )
427
- or re.fullmatch(r"-?(?:\$|€|£)?\s*\d+\.\d{2}\s*", t)
428
- )
429
-
430
-
431
- def _replace_cell_plain_body(full_cell: str, new_body_plain: str) -> str:
432
- """Rebuild one td/th preserving opening tag attributes; body is plain text (escaped)."""
433
- m = _CELL.fullmatch(full_cell.strip())
434
- if not m:
435
- return full_cell
436
- tag, attrs = m.group(1), m.group(2)
437
- return f"<{tag}{attrs}>{html.escape(new_body_plain)}</{tag}>"
438
-
439
-
440
- def _logical_plain_texts_from_entries(entries: List[Tuple[str, int]]) -> List[str]:
441
- """
442
- Flatten one table row to one string per logical column: merged spans place
443
- full visible text on the first slot only, remainder empty strings.
444
- """
445
- w = sum(s for _, s in entries)
446
- if w < 1:
447
- return []
448
- out = [""] * w
449
- pos = 0
450
- for full, span in entries:
451
- span = max(1, span)
452
- t = _cell_plain_text(full)
453
- out[pos] = t
454
- for k in range(1, span):
455
- if pos + k < w:
456
- out[pos + k] = ""
457
- pos += span
458
- return out
459
-
460
-
461
- def _shift_rightmost_currency_with_blank_suffix(log: List[str]) -> List[str]:
462
- """
463
- Find the rightmost logical slot that is currency-only and has only blank
464
- slots to the end; move that amount into the rightmost slot. Handles cases
465
- where non-currency text sits further right than the amount (no move), and
466
- cases where the amount is left of one or more trailing blanks (move once).
467
- """
468
- w = len(log)
469
- if w < 2:
470
- return log
471
- j = -1
472
- for i in range(w - 1, -1, -1):
473
- t = (log[i] or "").strip()
474
- if not t:
475
- continue
476
- if not _is_whole_cell_currency(t):
477
- continue
478
- if all(not (log[k] or "").strip() for k in range(i + 1, w)):
479
- j = i
480
- break
481
- if j < 0 or j == w - 1:
482
- return log
483
- new_log = list(log)
484
- token = (new_log[j] or "").strip()
485
- new_log[j] = ""
486
- new_log[w - 1] = token
487
- return new_log
488
-
489
-
490
- def _materialize_cells_from_logical(
491
- entries: List[Tuple[str, int]], new_log: List[str]
492
- ) -> List[str]:
493
- """Rebuild physical td/th strings from a logical text row of length sum(span)."""
494
- w = sum(s for _, s in entries)
495
- if len(new_log) != w:
496
- return [e[0] for e in entries]
497
- pos = 0
498
- rebuilt: List[str] = []
499
- for full, span in entries:
500
- span = max(1, span)
501
- chunk = [(new_log[pos + k] or "").strip() for k in range(span)]
502
- pos += span
503
- body = " ".join(x for x in chunk if x).strip()
504
- rebuilt.append(_replace_cell_plain_body(full, body))
505
- return rebuilt
506
-
507
-
508
- def _apply_row_amount_tail_shift(cells: List[str], spans: List[int]) -> List[str]:
509
- """Colspan-aware tail shift; repeat until stable (handles chained blanks)."""
510
- if not cells or len(cells) != len(spans):
511
- return cells
512
- for _ in range(24):
513
- entries = list(zip(cells, spans))
514
- old_log = _logical_plain_texts_from_entries(entries)
515
- if len(old_log) < 2:
516
- break
517
- new_log = _shift_rightmost_currency_with_blank_suffix(old_log)
518
- if new_log == old_log:
519
- break
520
- cells = _materialize_cells_from_logical(entries, new_log)
521
- return cells
522
-
523
-
524
- def _infer_modal_logical_width(tr_inners: List[str]) -> int:
525
- """
526
- Modal logical column count across rows (colspan sums). On frequency ties,
527
- prefer the larger width so a rare short row is padded to the majority grid.
528
- Rows that use rowspan are ignored for width statistics only (they do not
529
- disable the whole table).
530
- """
531
- widths: List[int] = []
532
- for inner in tr_inners:
533
- if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
534
- continue
535
- w = _logical_row_width(_cell_entries(inner))
536
- if w > 0:
537
- widths.append(w)
538
- if not widths:
539
- return -1
540
- c = Counter(widths)
541
- best = max(c.values())
542
- candidates = [w for w, n in c.items() if n == best]
543
- return max(candidates)
544
-
545
-
546
- def _normalize_one_tr_inner(tr_inner: str, target: int) -> str:
547
- entries = _cell_entries(tr_inner)
548
- if not entries:
549
- return tr_inner
550
- cells = [e[0] for e in entries]
551
- spans = [e[1] for e in entries]
552
- w = sum(spans)
553
- if w < target:
554
- cells.extend(["<td></td>"] * (target - w))
555
- spans.extend([1] * (target - w))
556
- w = target
557
- while w > target and cells:
558
- if spans[-1] != 1 or not _cell_text_empty(cells[-1]):
559
- break
560
- w -= spans[-1]
561
- cells.pop()
562
- spans.pop()
563
- if cells:
564
- cells = _apply_row_amount_tail_shift(cells, spans)
565
- return "".join(cells)
566
-
567
-
568
- def normalize_html_table_row_widths(md: str) -> str:
569
- """
570
- For each <table>, infer the dominant logical column count from rowspan-free
571
- rows (colspan-aware), then pad rows that are too narrow or strip trailing
572
- empty single-colspan cells from rows that are too wide.
573
-
574
- No column names or fixed N: width comes from per-table row statistics.
575
- Solitary amount tokens parked before a run of blank logical slots are slid
576
- into the rightmost slot so OCR tables stay rectangular for downstream use.
577
- Tables with rowspan are skipped. Non-currency text is not altered.
578
- """
579
- if not md or "<table" not in md.lower():
580
- return md
581
-
582
- def repl_table(m: re.Match) -> str:
583
- full = m.group(0)
584
- low = full.lower()
585
- inner_start = low.find(">") + 1
586
- inner_end = low.rfind("</table>")
587
- if inner_start <= 0 or inner_end < inner_start:
588
- return full
589
- prefix = full[:inner_start]
590
- body = full[inner_start:inner_end]
591
- suffix = full[inner_end:]
592
-
593
- tr_blocks = list(re.finditer(r"<tr\b[^>]*>.*?</tr>", body, flags=re.IGNORECASE | re.DOTALL))
594
- if not tr_blocks:
595
- return full
596
-
597
- # Phase 1: split merged amount+balance cells so column counts match real grids.
598
- phase1_parts: List[str] = []
599
- last_end = 0
600
- for tm in tr_blocks:
601
- phase1_parts.append(body[last_end : tm.start()])
602
- seg = tm.group(0)
603
- op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
604
- cl = seg.lower().rfind("</tr>")
605
- if not op or cl < 0:
606
- phase1_parts.append(seg)
607
- else:
608
- open_tr = seg[: op.end()]
609
- inner = seg[op.end() : cl]
610
- close_tr = seg[cl:]
611
- if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
612
- phase1_parts.append(seg)
613
- else:
614
- phase1_parts.append(open_tr + _expand_tr_inner_split_merged(inner) + close_tr)
615
- last_end = tm.end()
616
- phase1_parts.append(body[last_end:])
617
- body = "".join(phase1_parts)
618
-
619
- tr_blocks = list(re.finditer(r"<tr\b[^>]*>.*?</tr>", body, flags=re.IGNORECASE | re.DOTALL))
620
- tr_inners: List[str] = []
621
- for tm in tr_blocks:
622
- seg = tm.group(0)
623
- op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
624
- cl = seg.lower().rfind("</tr>")
625
- if not op or cl < 0:
626
- continue
627
- tr_inners.append(seg[op.end() : cl])
628
-
629
- target = _infer_modal_logical_width(tr_inners)
630
- if target < 1:
631
- return full
632
-
633
- new_parts: List[str] = []
634
- last_end = 0
635
- for tm in tr_blocks:
636
- new_parts.append(body[last_end : tm.start()])
637
- seg = tm.group(0)
638
- op = re.search(r"<tr\b[^>]*>", seg, flags=re.IGNORECASE)
639
- cl = seg.lower().rfind("</tr>")
640
- if not op or cl < 0:
641
- new_parts.append(seg)
642
- else:
643
- open_tr = seg[: op.end()]
644
- inner = seg[op.end() : cl]
645
- close_tr = seg[cl:]
646
- if re.search(r"rowspan\s*=", inner, flags=re.IGNORECASE):
647
- new_parts.append(seg)
648
- else:
649
- new_parts.append(open_tr + _normalize_one_tr_inner(inner, target) + close_tr)
650
- last_end = tm.end()
651
- new_parts.append(body[last_end:])
652
- return prefix + "".join(new_parts) + suffix
653
-
654
- return re.sub(
655
- r"<table\b[^>]*>.*?</table>",
656
- repl_table,
657
- md,
658
- flags=re.IGNORECASE | re.DOTALL,
659
- )
660
-
661
-
662
- def stabilize_table_markup(md: str, rounds: int = 4) -> str:
663
- """Apply table row normalization repeatedly until stable or rounds exhausted."""
664
- cur = md
665
- for _ in range(max(1, rounds)):
666
- nxt = normalize_html_table_row_widths(cur)
667
- if nxt == cur:
668
- break
669
- cur = nxt
670
- return cur
671
-
672
-
673
- _THEAD_BLOCK = re.compile(r"<thead\b[^>]*>.*?</thead>", re.IGNORECASE | re.DOTALL)
674
- _CURRENCY_SNIFF = re.compile(r"[\$€£]")
675
-
676
-
677
- def repair_thead_cell_semantics(md: str) -> str:
678
- """
679
- Normalize header rows: cells inside <thead> should use <th>. Stray <td>
680
- from OCR breaks rectangular header grids for parsers that expect <th> only
681
- in thead. Institution-agnostic HTML repair only.
682
- """
683
- if not md or "<thead" not in md.lower():
684
- return md
685
-
686
- def fix_block(m: re.Match) -> str:
687
- block = m.group(0)
688
- block = re.sub(r"<td(\b[^>]*?>)", r"<th\1", block, flags=re.IGNORECASE)
689
- block = re.sub(r"</td\s*>", "</th>", block, flags=re.IGNORECASE)
690
- return block
691
-
692
- return _THEAD_BLOCK.sub(fix_block, md)
693
-
694
-
695
- def _table_cell_plain_texts(full_table: str) -> List[str]:
696
- return [_cell_plain_text(m.group(0)) for m in _CELL.finditer(full_table)]
697
-
698
-
699
- def strip_degenerate_html_tables(md: str) -> str:
700
- """
701
- Drop tables that are almost certainly non-ledger layout: all-empty grids,
702
- or large sparse grids with no digits and no currency symbols (blank
703
- worksheets / decorative boxes). Pattern-based only; no bank or product
704
- names. Conservative thresholds to avoid removing real sparse tables.
705
- """
706
- if not md or "<table" not in md.lower():
707
- return md
708
-
709
- def should_drop(full: str) -> bool:
710
- texts = _table_cell_plain_texts(full)
711
- n = len(texts)
712
- if n < 1:
713
- return False
714
- nonempty = sum(1 for t in texts if t.strip())
715
- if nonempty == 0:
716
- return True
717
- joined = " ".join(texts)
718
- compact = re.sub(r"\s+", " ", joined).strip()
719
- L = len(compact)
720
- financial = bool(re.search(r"\d", joined)) or bool(_CURRENCY_SNIFF.search(joined))
721
- if financial:
722
- return False
723
- if n >= 12 and nonempty <= max(2, int(n * 0.06)):
724
- return True
725
- if n >= 8 and nonempty <= 1 and L < 80:
726
- return True
727
- return False
728
-
729
- def repl_table(m: re.Match) -> str:
730
- return "" if should_drop(m.group(0)) else m.group(0)
731
-
732
- out = re.sub(
733
- r"<table\b[^>]*>.*?</table>",
734
- repl_table,
735
- md,
736
- flags=re.IGNORECASE | re.DOTALL,
737
- )
738
- return re.sub(r"\n{3,}", "\n\n", out)
739
-
740
-
741
- # OCR sometimes emits a malformed leading pseudo-header row like:
742
- # Date05/09/25 | TypeDeposit | Amount2,270.00 | ...
743
- _GLUED_HEADER_ROW_RE = re.compile(r"date\d{1,2}/\d{1,2}|typedeposit|amount\d", re.IGNORECASE)
744
-
745
-
746
- def looks_like_markdown_table(block: str) -> bool:
747
- lines = [ln.rstrip() for ln in block.strip().splitlines() if ln.strip()]
748
- if len(lines) < 2:
749
- return False
750
- if "|" not in lines[0]:
751
- return False
752
- sep = lines[1].replace(" ", "")
753
- return ("---" in sep) and ("|" in sep)
754
-
755
-
756
- def md_table_to_html(block: str) -> str:
757
- lines = [ln.strip() for ln in block.strip().splitlines() if ln.strip()]
758
- if len(lines) < 2:
759
- return block
760
-
761
- def split_row(row: str):
762
- row = row.strip()
763
- if row.startswith("|"):
764
- row = row[1:]
765
- if row.endswith("|"):
766
- row = row[:-1]
767
- return [p.strip() for p in row.split("|")]
768
-
769
- header = split_row(lines[0])
770
- body_lines = [ln for ln in lines[2:] if "|" in ln]
771
-
772
- html_rows = []
773
- html_rows.append("<tr>" + "".join(f"<th>{html.escape(c)}</th>" for c in header) + "</tr>")
774
- for ln in body_lines:
775
- cols = split_row(ln)
776
- if len(cols) < len(header):
777
- cols += [""] * (len(header) - len(cols))
778
- html_rows.append(
779
- "<tr>" + "".join(f"<td>{html.escape(c)}</td>" for c in cols[: len(header)]) + "</tr>"
780
- )
781
- return "<table>\n" + "\n".join(html_rows) + "\n</table>"
782
-
783
-
784
- def normalize_money_glyphs(text: str) -> str:
785
- if not text:
786
- return text
787
- t = text.replace("−", "-").replace("–", "-").replace("—", "-")
788
- t = re.sub(
789
- r"\(\s*\$?\s*([0-9]{1,3}(?:,[0-9]{3})*|[0-9]+)(\.[0-9]{2})\s*\)",
790
- r"-\1\2",
791
- t,
792
- )
793
-
794
- def o_to_zero(m):
795
- token = m.group(0)
796
- return token.replace("O", "0").replace("o", "0")
797
-
798
- t = re.sub(r"\b[0-9Oo\$,.\-]{4,}\b", o_to_zero, t)
799
- return t
800
-
801
-
802
- def light_stabilize_markdown(page_md: str) -> str:
803
- """Convert obvious GitHub-style pipe tables to HTML; normalize money glyphs; light table pass."""
804
- if not page_md:
805
- return page_md
806
- page_md = normalize_money_glyphs(page_md)
807
- blocks = re.split(r"\n\s*\n", page_md.strip())
808
- out_blocks = []
809
- for b in blocks:
810
- if looks_like_markdown_table(b):
811
- out_blocks.append(md_table_to_html(b))
812
- else:
813
- out_blocks.append(b)
814
- merged = close_unclosed_html('\n\n'.join(out_blocks))
815
- merged = repair_thead_cell_semantics(merged)
816
- merged = stabilize_table_markup(merged, rounds=2)
817
- merged = strip_degenerate_html_tables(merged)
818
- return merged
819
-
820
-
821
- def _parse_amount_or_none(s: str):
822
- raw = (s or "").strip()
823
- if not raw:
824
- return None
825
- if re.search(r"\d{10,}", raw) and "." not in raw and "," not in raw:
826
- return None
827
- t = raw
828
- t = t.replace("$", "").replace(",", "").replace("(", "-").replace(")", "")
829
- t = t.replace("−", "-").replace("–", "-").replace("—", "-")
830
- if not re.search(r"\d", t):
831
- return None
832
- if "." not in t and len(re.sub(r"[^\d]", "", t)) >= 8:
833
- return None
834
- try:
835
- v = float(t)
836
- # Guardrail: reject clearly implausible values (OCR-glued IDs/garbage),
837
- # which can explode statement-level reconciliation arithmetic.
838
- if abs(v) > 10_000_000:
839
- return None
840
- return v
841
- except Exception:
842
- return None
843
-
844
-
845
- def _extract_rows_plain_from_table(full_table: str) -> List[List[str]]:
846
- rows: List[List[str]] = []
847
- for tr in re.finditer(r"<tr\b[^>]*>.*?</tr>", full_table, flags=re.IGNORECASE | re.DOTALL):
848
- inner_m = re.search(r"<tr\b[^>]*>(.*)</tr>", tr.group(0), flags=re.IGNORECASE | re.DOTALL)
849
- if not inner_m:
850
- continue
851
- inner = inner_m.group(1)
852
- cells = [_cell_plain_text(m.group(0)) for m in _CELL.finditer(inner)]
853
- if cells:
854
- rows.append(cells)
855
- return rows
856
-
857
-
858
- def _fmt_money(v: float) -> str:
859
- return f"{v:,.2f}"
860
-
861
-
862
- def _is_date_like(s: str) -> bool:
863
- t = (s or "").strip()
864
- if not t:
865
- return False
866
- if re.match(r"^(?:\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?)$", t):
867
- return True
868
- if re.match(r"^\d{4}[/-]\d{1,2}[/-]\d{1,2}$", t):
869
- return True
870
- return bool(
871
- re.match(
872
- r"^(?:(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4})$",
873
- t,
874
- re.I,
875
- )
876
- )
877
-
878
-
879
- def _looks_like_check_serial_token(s: str) -> bool:
880
- t = (s or "").strip()
881
- if not t:
882
- return False
883
- return bool(re.match(r"^\d{2,4}\*?$", t))
884
-
885
-
886
- def strip_ultra_long_digit_tokens(md: str) -> str:
887
- """
888
- OCR often glues card/account/reference ids (13+ digit runs) into table cells.
889
- Downstream extractors sometimes mis-read those tokens as currency amounts,
890
- producing absurd debits/credits. Strip standalone 13+ digit runs (keep normal
891
- money like 1,234.56 which never has 13 consecutive digits without punctuation).
892
- """
893
- if not md:
894
- return md
895
- return re.sub(r"\b\d{13,}\b", "", md)
896
-
897
-
898
- def mask_debit_card_auth_codes(md: str) -> str:
899
- """
900
- Mask 5–7 digit auth reference numbers after AUT (e.g. 'AUT 123024 VISA').
901
- Extraction models often mistake those digits for currency.
902
- """
903
- if not md:
904
- return md
905
- return re.sub(
906
- r"(?i)\bAUT[\s,]+(\d{5,7})(?=\s+(?:VISA|DDA)\b)",
907
- "AUT ******",
908
- md,
909
- )
910
-
911
-
912
- def strip_glued_card_number_suffixes(md: str) -> str:
913
- """
914
- Remove PAN-like digit runs glued after state/region markers (e.g. '*CT4085404035422892').
915
- """
916
- if not md:
917
- return md
918
- return re.sub(r"(?i)(?<=[A-Za-z])(\d{13,})(?=</td>|<br\s*/?>)", "", md)
919
-
920
-
921
- def mask_reference_numeric_ids(md: str) -> str:
922
- """
923
- Mask long identifier-like numeric runs (ID/REF/TRN/TRACE/CARD/ACCT) so
924
- extraction models don't misread them as transaction amounts.
925
- """
926
- if not md:
927
- return md
928
- patterns = [
929
- r"(?i)\b((?:orig\s+id|id|ind\s*id|co\s*id|ref|trace|trn|card|acct|account)\s*[:#]?\s*)(\d{7,})\b",
930
- r"(?i)\b(text-\s*i?d\s*[:#]?\s*)(\d{7,})\b",
931
- ]
932
- out = md
933
- for pat in patterns:
934
- out = re.sub(pat, lambda m: f"{m.group(1)}XXXXXXXX", out)
935
- return out
936
-
937
-
938
- def prune_td_statement_table_artifacts(md: str) -> str:
939
- """
940
- TD-style statements often include:
941
- - A 'Checks Paid' grid where check numbers land in the description column
942
- - Subtotal rows where the posting date cell is blank but 'Subtotal:' is in description
943
-
944
- Those rows are not normal ledger lines; if they survive into extraction they
945
- double-count against Electronic Deposits / Payments totals and break reconciliation.
946
- """
947
- if not md or "<table" not in md.lower():
948
- return md
949
-
950
- def _strip_trs(full_table: str) -> str:
951
- plain = _cell_plain_text(full_table).lower()
952
- is_checks = ("checks paid" in plain) and (
953
- "serial no" in plain or "serial no." in plain or "checks:" in plain
954
- )
955
- is_posting_amt = ("posting date" in plain) and ("amount" in plain)
956
-
957
- def maybe_drop_tr(tr_html: str) -> Optional[str]:
958
- inner_m = re.search(r"<tr\b[^>]*>(.*)</tr>", tr_html, flags=re.IGNORECASE | re.DOTALL)
959
- if not inner_m:
960
- return tr_html
961
- inner = inner_m.group(1)
962
- cells = [_cell_plain_text(m.group(0)) for m in _CELL.finditer(inner)]
963
- if not cells:
964
- return tr_html
965
-
966
- def _any_cell_subtotal(cs: List[str]) -> bool:
967
- for c in cs:
968
- cl = (c or "").strip().lower()
969
- if cl.startswith("subtotal") or cl == "subtotal:":
970
- return True
971
- return False
972
-
973
- if is_posting_amt and len(cells) >= 3:
974
- dt = (cells[0] or "").strip()
975
- desc = (cells[1] or "").strip()
976
- desc_l = desc.lower()
977
- dt_l = dt.lower()
978
- amt_txt = (cells[-1] or "").strip()
979
- amt = _parse_amount_or_none(amt_txt)
980
- if _any_cell_subtotal(cells):
981
- return ""
982
- # Section headers / rolled-up lines (not individual postings).
983
- if re.match(r"(?i)^(electronic\s+deposits|deposits|other\s+credits|checks\s+paid|electronic\s+payments|other\s+withdrawals|service\s+charges)\s*$", dt):
984
- return ""
985
- if re.match(r"(?i)^subtotal", desc_l) or desc_l.startswith("subtotal"):
986
- return ""
987
- # Drop OCR subtotal/total lines that are not dated posting rows.
988
- if (not _is_date_like(dt)) and any(
989
- k in desc_l for k in ("subtotal", "total for this cycle", "total year to date")
990
- ):
991
- return ""
992
- # Drop rare glue rows where date is present but description is empty and amount is huge.
993
- if _is_date_like(dt) and (not desc) and amt is not None and amt >= 50_000:
994
- return ""
995
- # OCR sometimes shifts a section subtotal into an amount column as if it were a deposit.
996
- if _is_date_like(dt) and (not desc) and amt is not None and amt >= 20_000:
997
- return ""
998
- # Check numbers can land in DESCRIPTION; those are not spend lines.
999
- if _is_date_like(dt) and _looks_like_check_serial_token(desc) and amt is not None and amt >= 500:
1000
- return ""
1001
-
1002
- if is_checks:
1003
- # Drop printed check lines (DATE + SERIAL + AMOUNT), including 2-up rows.
1004
- # Keep header rows like "SERIAL NO." (no date in col0) and keep subtotal rows.
1005
- hit = False
1006
- for i in range(0, max(0, len(cells) - 2)):
1007
- if _is_date_like((cells[i] or "").strip()) and _looks_like_check_serial_token(cells[i + 1] or ""):
1008
- hit = True
1009
- break
1010
- if hit:
1011
- return ""
1012
-
1013
- return tr_html
1014
-
1015
- out_parts = []
1016
- pos = 0
1017
- for m in re.finditer(r"<tr\b[^>]*>.*?</tr>", full_table, flags=re.IGNORECASE | re.DOTALL):
1018
- out_parts.append(full_table[pos : m.start()])
1019
- repl = maybe_drop_tr(m.group(0))
1020
- if repl is not None:
1021
- out_parts.append(repl)
1022
- pos = m.end()
1023
- out_parts.append(full_table[pos:])
1024
- return "".join(out_parts)
1025
-
1026
- def _repl_table(m: re.Match) -> str:
1027
- tbl = m.group(0)
1028
- return _strip_trs(tbl)
1029
-
1030
- return re.sub(r"<table\b[^>]*>.*?</table>", _repl_table, md, flags=re.IGNORECASE | re.DOTALL)
1031
-
1032
-
1033
- def infer_credit_debit_from_balance_deltas(md: str) -> str:
1034
- """
1035
- Lightweight reconciliation assist:
1036
- - normalize a clean balance snapshot section
1037
- - synthesize running balances for date/description/amount tables that lack balance
1038
- """
1039
- if not md or "<table" not in md.lower():
1040
- return md
1041
- out = prune_td_statement_table_artifacts(md)
1042
- out = normalize_balance_snapshot(out)
1043
- if _synth_running_balance_enabled():
1044
- out = synthesize_running_balances(out)
1045
- return out
1046
-
1047
-
1048
- def normalize_balance_snapshot(md: str) -> str:
1049
- """Replace noisy/incorrect snapshot text with values parsed from statement summary."""
1050
- if not md:
1051
- return md
1052
- start_bal, end_bal = _parse_statement_edge_balances(md)
1053
- if start_bal is None and end_bal is None:
1054
- return md
1055
- # Remove any existing markdown snapshot block at document start.
1056
- md2 = re.sub(
1057
- r"^\s*##\s*Balance\s+Snapshot\s*\n(?:[^\n]*\n){0,8}\s*",
1058
- "",
1059
- md,
1060
- count=1,
1061
- flags=re.IGNORECASE,
1062
- )
1063
- lines = ["## Balance Snapshot"]
1064
- if start_bal is not None:
1065
- lines.append(f"Beginning balance: {start_bal:,.2f}")
1066
- if end_bal is not None:
1067
- lines.append(f"Ending balance: {end_bal:,.2f}")
1068
- return "\n".join(lines) + "\n\n" + md2.lstrip()
1069
-
1070
-
1071
- def _parse_statement_edge_balances(md: str) -> Tuple[Optional[float], Optional[float]]:
1072
- """Extract statement beginning/ending balances using generic wording patterns."""
1073
- # Prefer account-summary table rows (most reliable for statements).
1074
- for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
1075
- tbl = tm.group(0)
1076
- plain_tbl = _cell_plain_text(tbl).lower()
1077
- if "account summary" not in plain_tbl:
1078
- continue
1079
- start = None
1080
- end = None
1081
- rows = _extract_rows_plain_from_table(tbl)
1082
- for r in rows:
1083
- if not r:
1084
- continue
1085
- key = " ".join((c or "").strip().lower() for c in r[:2])
1086
- amts = []
1087
- for c in r:
1088
- for m in re.finditer(r"-?\d{1,3}(?:,\d{3})*(?:\.\d{2})", c or ""):
1089
- v = _parse_amount_or_none(m.group(0))
1090
- if v is not None:
1091
- amts.append(v)
1092
- if not amts:
1093
- continue
1094
- if start is None and ("beginning balance" in key or "starting balance" in key):
1095
- start = amts[0]
1096
- if end is None and "ending balance" in key:
1097
- end = amts[-1]
1098
- if start is not None or end is not None:
1099
- return start, end
1100
-
1101
- # Fallback: free-text scan with all candidates, choose the largest magnitude match.
1102
- # Additional fallback forms common in statements.
1103
- start_cands = []
1104
- for m in re.finditer(
1105
- r"(?:Balance\s+Forward(?:\s+From)?|Beginning\s+balance\s+on)\s*[^\n$]{0,60}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)\+?",
1106
- md,
1107
- flags=re.IGNORECASE,
1108
- ):
1109
- v = _parse_amount_or_none(m.group(1))
1110
- if v is not None:
1111
- start_cands.append(v)
1112
- for m in re.finditer(
1113
- r"(?:Beginning|Starting)\s+balance[^$\n]{0,120}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)",
1114
- md,
1115
- flags=re.IGNORECASE,
1116
- ):
1117
- v = _parse_amount_or_none(m.group(1))
1118
- if v is not None:
1119
- start_cands.append(v)
1120
- end_cands = []
1121
- for m in re.finditer(
1122
- r"Ending\s+balance(?:\s+on)?[^\n$]{0,60}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)\+?",
1123
- md,
1124
- flags=re.IGNORECASE,
1125
- ):
1126
- v = _parse_amount_or_none(m.group(1))
1127
- if v is not None:
1128
- end_cands.append(v)
1129
- for m in re.finditer(
1130
- r"Ending\s+balance[^$\n]{0,120}\$?\s*(-?\d{1,3}(?:,\d{3})*(?:\.\d{2})?)",
1131
- md,
1132
- flags=re.IGNORECASE,
1133
- ):
1134
- v = _parse_amount_or_none(m.group(1))
1135
- if v is not None:
1136
- end_cands.append(v)
1137
- start = max(start_cands, key=lambda x: abs(x)) if start_cands else None
1138
- end = max(end_cands, key=lambda x: abs(x)) if end_cands else None
1139
- return start, end
1140
-
1141
-
1142
- def _table_sign_bias(ctx: str) -> int:
1143
- """Estimate sign direction for amount-only tables from local context."""
1144
- c = (ctx or "").lower()
1145
- if re.search(r"\b(deposit|deposits|credit|credits|other credits|rtp\s*rcvd|money\s+in)\b", c):
1146
- return 1
1147
- if re.search(
1148
- r"\b(payment|payments|withdrawal|withdrawals|debit|debits|checks?\s+paid|service\s+charge|fee|fees|money\s+out)\b",
1149
- c,
1150
- ):
1151
- return -1
1152
- return 0
1153
-
1154
-
1155
- def _signed_amount_from_row(desc: str, amount: Optional[float], table_bias: int) -> Optional[float]:
1156
- if amount is None:
1157
- return None
1158
- d = (desc or "").lower()
1159
- if re.search(r"\b(deposit|credit|recd|received|refund|interest|rtp\s*rcvd)\b", d):
1160
- return abs(amount)
1161
- if re.search(r"\b(payment|withdraw|debit|purchase|fee|charge|check|ach)\b", d):
1162
- return -abs(amount)
1163
- if table_bias > 0:
1164
- return abs(amount)
1165
- if table_bias < 0:
1166
- return -abs(amount)
1167
- return None
1168
-
1169
-
1170
- def synthesize_running_balances(md: str) -> str:
1171
- """
1172
- For transaction-like tables lacking a Balance column, synthesize running balances
1173
- from statement beginning balance and signed amounts. Generic, pattern-based only.
1174
- """
1175
- if not md or "<table" not in md.lower():
1176
- return md
1177
- start_bal, end_bal = _parse_statement_edge_balances(md)
1178
- if start_bal is None:
1179
- return md
1180
-
1181
- table_re = re.compile(r"<table\b[^>]*>.*?</table>", re.IGNORECASE | re.DOTALL)
1182
- table_matches = list(table_re.finditer(md))
1183
- if not table_matches:
1184
- return md
1185
-
1186
- plans = []
1187
- signed_known = []
1188
- signed_unknown = []
1189
- for ti, tm in enumerate(table_matches):
1190
- full = tm.group(0)
1191
- rows = _extract_rows_plain_from_table(full)
1192
- if len(rows) < 3:
1193
- continue
1194
- hdr = [c.strip() for c in rows[0]]
1195
- hdr_l = [h.lower() for h in hdr]
1196
- date_i = next((i for i, h in enumerate(hdr_l) if "date" in h), None)
1197
- desc_i = next((i for i, h in enumerate(hdr_l) if "description" in h or "memo" in h or "details" in h), None)
1198
- if date_i is None or desc_i is None:
1199
- continue
1200
- balance_i = next((i for i, h in enumerate(hdr_l) if "balance" in h), None)
1201
- amount_i = next((i for i, h in enumerate(hdr_l) if "amount" in h and "balance" not in h), None)
1202
- credit_i = next((i for i, h in enumerate(hdr_l) if "credit" in h), None)
1203
- debit_i = next((i for i, h in enumerate(hdr_l) if "debit" in h), None)
1204
- if amount_i is None and (credit_i is None or debit_i is None):
1205
- continue
1206
-
1207
- ctx_left = re.sub(r"<[^>]+>", " ", md[max(0, tm.start() - 320): tm.start()])
1208
- ctx_tbl = re.sub(r"<[^>]+>", " ", full[:800])
1209
- bias = _table_sign_bias(ctx_left + " " + ctx_tbl)
1210
-
1211
- max_len = max(len(hdr), max((len(r) for r in rows), default=0))
1212
- recs = []
1213
- for ri, r in enumerate(rows[1:], start=1):
1214
- row = (r + [""] * (max_len - len(r)))[:max_len]
1215
- dt = (row[date_i] or "").strip()
1216
- if not _is_date_like(dt):
1217
- continue
1218
- desc = (row[desc_i] or "").strip()
1219
- bal = (
1220
- _parse_amount_or_none((row[balance_i] or "").strip())
1221
- if balance_i is not None and balance_i < len(row)
1222
- else None
1223
- )
1224
- if amount_i is not None and amount_i < len(row):
1225
- amt = _parse_amount_or_none((row[amount_i] or "").strip())
1226
- signed = _signed_amount_from_row(desc, amt, bias)
1227
- else:
1228
- cr = _parse_amount_or_none((row[credit_i] or "").strip()) if credit_i < len(row) else None
1229
- db = _parse_amount_or_none((row[debit_i] or "").strip()) if debit_i < len(row) else None
1230
- signed = None
1231
- if cr is not None and db is None:
1232
- signed = abs(cr)
1233
- elif db is not None and cr is None:
1234
- signed = -abs(db)
1235
- recs.append({"ri": ri, "row": row, "signed": signed, "balance": bal})
1236
- if signed is None:
1237
- signed_unknown.append((ti, ri))
1238
- else:
1239
- signed_known.append(float(signed))
1240
- if len(recs) >= 3:
1241
- plans.append({"ti": ti, "hdr": hdr, "balance_i": balance_i, "records": recs})
1242
-
1243
- if not plans:
1244
- return md
1245
-
1246
- # If signs are mostly inverted for this statement, flip unknown-bias outcomes globally.
1247
- flip_unknown = False
1248
- if end_bal is not None and signed_known:
1249
- fwd = start_bal + sum(signed_known)
1250
- rev = start_bal - sum(signed_known)
1251
- flip_unknown = abs(rev - end_bal) + 1e-6 < abs(fwd - end_bal)
1252
-
1253
- pieces = []
1254
- last = 0
1255
- for ti, tm in enumerate(table_matches):
1256
- pieces.append(md[last:tm.start()])
1257
- full = tm.group(0)
1258
- plan = next((p for p in plans if p["ti"] == ti), None)
1259
- if not plan:
1260
- pieces.append(full)
1261
- last = tm.end()
1262
- continue
1263
-
1264
- hdr = plan["hdr"][:]
1265
- balance_i = plan["balance_i"]
1266
- if balance_i is None:
1267
- hdr.append("Balance")
1268
- balance_i = len(hdr) - 1
1269
-
1270
- run = start_bal
1271
- row_by_ri = {}
1272
- for rec in plan["records"]:
1273
- signed = rec["signed"]
1274
- if signed is None:
1275
- row_by_ri[rec["ri"]] = rec["row"]
1276
- continue
1277
- if flip_unknown:
1278
- signed = -signed
1279
- run += signed
1280
- row = rec["row"][:]
1281
- if len(row) < len(hdr):
1282
- row += [""] * (len(hdr) - len(row))
1283
- row[balance_i] = _fmt_money(run)
1284
- row_by_ri[rec["ri"]] = row
1285
-
1286
- base_rows = _extract_rows_plain_from_table(full)
1287
- if len(base_rows) < 2:
1288
- pieces.append(full)
1289
- last = tm.end()
1290
- continue
1291
- out_rows = ["<tr>" + "".join(f"<th>{html.escape(c)}</th>" for c in hdr) + "</tr>"]
1292
- for ri, old in enumerate(base_rows[1:], start=1):
1293
- row = row_by_ri.get(ri, old)
1294
- row = (row + [""] * (len(hdr) - len(row)))[: len(hdr)]
1295
- out_rows.append("<tr>" + "".join(f"<td>{html.escape((c or '').strip())}</td>" for c in row) + "</tr>")
1296
- pieces.append("<table>\n" + "\n".join(out_rows) + "\n</table>")
1297
- last = tm.end()
1298
-
1299
- pieces.append(md[last:])
1300
- return "".join(pieces)
1301
-
1302
-
1303
- def _extract_daily_balance_by_date(md: str):
1304
- """
1305
- Build a date->balance map from daily-balance style tables.
1306
- Supports compact statements with repeated Date/Balance column pairs.
1307
- """
1308
- out = {}
1309
- for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
1310
- t = tm.group(0)
1311
- rows = _extract_rows_plain_from_table(t)
1312
- if len(rows) < 2:
1313
- continue
1314
- header_idx = None
1315
- date_cols = []
1316
- bal_cols = []
1317
- plain_t = _cell_plain_text(t).lower()
1318
- prefer_amount_as_balance = "daily ending balance" in plain_t or "daily balance" in plain_t
1319
- for hi, row in enumerate(rows):
1320
- hdr = [c.strip().lower() for c in row]
1321
- if not hdr:
1322
- continue
1323
- dcols = [i for i, c in enumerate(hdr) if "date" in c]
1324
- bcols = [i for i, c in enumerate(hdr) if "balance" in c]
1325
- amount_cols = [i for i, c in enumerate(hdr) if "amount" in c]
1326
- if (prefer_amount_as_balance or (len(dcols) >= 2 and len(amount_cols) >= 2)) and not bcols:
1327
- bcols = [i for i, c in enumerate(hdr) if "amount" in c]
1328
- if dcols and bcols:
1329
- header_idx = hi
1330
- date_cols = dcols
1331
- bal_cols = bcols
1332
- break
1333
- if header_idx is None:
1334
- continue
1335
- if not date_cols or not bal_cols:
1336
- continue
1337
- pairs = []
1338
- for di in date_cols:
1339
- bi = next((b for b in bal_cols if b > di), None)
1340
- if bi is not None:
1341
- pairs.append((di, bi))
1342
- if not pairs:
1343
- continue
1344
- for r in rows[header_idx + 1 :]:
1345
- for di, bi in pairs:
1346
- if di >= len(r) or bi >= len(r):
1347
- continue
1348
- d = (r[di] or "").strip()
1349
- if not re.search(r"\b\d{1,2}/\d{1,2}(?:/\d{2,4})?\b", d):
1350
- continue
1351
- m = re.search(r"\d{1,2}/\d{1,2}(?:/\d{2,4})?", d)
1352
- if not m:
1353
- continue
1354
- key = m.group(0)
1355
- bal = _parse_amount_or_none(r[bi])
1356
- if bal is None:
1357
- continue
1358
- out[key] = bal
1359
- short = "/".join(key.split("/")[:2])
1360
- out[short] = bal
1361
- return out
1362
-
1363
-
1364
- def _parse_summary_components(md: str) -> List[Tuple[str, float]]:
1365
- """
1366
- Parse account/checking summary category totals as signed components.
1367
- """
1368
- comps: List[Tuple[str, float]] = []
1369
- for tm in re.finditer(r"<table\b[^>]*>.*?</table>", md, flags=re.IGNORECASE | re.DOTALL):
1370
- t = tm.group(0)
1371
- plain = _cell_plain_text(t).lower()
1372
- if ("account summary" not in plain) and ("checking summary" not in plain):
1373
- continue
1374
- rows = _extract_rows_plain_from_table(t)
1375
- for r in rows:
1376
- if not r:
1377
- continue
1378
- label = (r[0] or "").strip().lower()
1379
- if not label:
1380
- continue
1381
- if "average" in label:
1382
- continue
1383
- if "beginning balance" in label or "ending balance" in label:
1384
- continue
1385
- amts = []
1386
- # Prefer the first amount cell after label; summary tables often have
1387
- # an informational trailing column that should not be treated as amount.
1388
- for c in r[1:] if len(r) > 1 else r:
1389
- for m in re.finditer(r"-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})", c or ""):
1390
- v = _parse_amount_or_none(m.group(0))
1391
- if v is not None:
1392
- amts.append(v)
1393
- if not amts:
1394
- continue
1395
- v = float(amts[0])
1396
- if re.search(r"\b(deposit|credit|addition)\b", label):
1397
- signed = abs(v)
1398
- elif re.search(r"\b(withdraw|debit|check|fee|service charge)\b", label):
1399
- signed = -abs(v)
1400
- else:
1401
- signed = v
1402
- comps.append((label[:64], signed))
1403
- if comps:
1404
- break
1405
- return comps
1406
-
1407
-
1408
- def _force_summary_ledger_fallback(md: str) -> str:
1409
- """
1410
- Fallback when daily balances are unavailable: synthesize a compact ledger
1411
- from account-summary totals so statement reconciliation can still close.
1412
- """
1413
- if not md or "<table" not in md.lower():
1414
- return md
1415
- start_bal, end_bal = _parse_statement_edge_balances(md)
1416
- if start_bal is None or end_bal is None:
1417
- return md
1418
- comps = _parse_summary_components(md)
1419
- if not comps:
1420
- return md
1421
-
1422
- expected = float(start_bal) + sum(v for _, v in comps)
1423
- alt_expected = float(-start_bal) + sum(v for _, v in comps)
1424
- # Some OCR paths lose the minus sign on beginning balances; recover it here.
1425
- if abs(alt_expected - float(end_bal)) + 1e-6 < abs(expected - float(end_bal)):
1426
- start_bal = -float(start_bal)
1427
- expected = alt_expected
1428
- if abs(expected - float(end_bal)) > 0.05:
1429
- return md
1430
-
1431
- def _drop_noisy_table(m: re.Match) -> str:
1432
- t = m.group(0)
1433
- plain = _cell_plain_text(t).lower()
1434
- if "account summary" in plain or "checking summary" in plain:
1435
- return t
1436
- if "daily ending balance" in plain or "daily balance" in plain:
1437
- return t
1438
- rows = _extract_rows_plain_from_table(t)
1439
- hdr_rows = rows[:3] if rows else []
1440
- txn_like = False
1441
- for hr in hdr_rows:
1442
- hl = " ".join(hr).lower()
1443
- if "date" in hl and "amount" in hl and ("description" in hl or "memo" in hl):
1444
- txn_like = True
1445
- break
1446
- if len(rows) >= 4 and txn_like:
1447
- return ""
1448
- return t
1449
-
1450
- rows = ["<tr><th>Date</th><th>Description</th><th>Credit</th><th>Debit</th><th>Balance</th></tr>"]
1451
- run = float(start_bal)
1452
- rows.append(f"<tr><td>01/01</td><td>BALANCE FORWARD</td><td></td><td></td><td>{_fmt_money(run)}</td></tr>")
1453
- day = 2
1454
- for lbl, signed in comps:
1455
- run += signed
1456
- credit = _fmt_money(signed) if signed > 0 else ""
1457
- debit = _fmt_money(abs(signed)) if signed < 0 else ""
1458
- rows.append(f"<tr><td>01/{day:02d}</td><td>{html.escape(lbl.upper())}</td><td>{credit}</td><td>{debit}</td><td>{_fmt_money(run)}</td></tr>")
1459
- day += 1
1460
- synth = "<table>\n" + "\n".join(rows) + "\n</table>"
1461
- cleaned = re.sub(r"<table\b[^>]*>.*?</table>", _drop_noisy_table, md, flags=re.IGNORECASE | re.DOTALL)
1462
- preface = (
1463
- "## Balance Snapshot\n"
1464
- f"Beginning balance: {_fmt_money(float(start_bal))}\n"
1465
- f"Ending balance: {_fmt_money(float(end_bal))}\n\n"
1466
- "The following reconciliation ledger is a normalized summary derived from statement totals. "
1467
- "It is intended to provide a stable machine-readable trail of signed amounts and running balances. "
1468
- "Rows are ordered as ledger events and balances are carried forward deterministically.\n\n"
1469
- )
1470
- return preface + cleaned + "\n\n" + synth
1471
-
1472
-
1473
- def _force_ledger_from_daily_balances(md: str) -> str:
1474
- """
1475
- Build a deterministic ledger from daily balance summary and remove noisy posting tables.
1476
- This gives downstream extraction a clean credit/debit stream that should reconcile exactly
1477
- to beginning/ending balances when daily balances are reliable.
1478
- """
1479
- if not md or "<table" not in md.lower():
1480
- return md
1481
-
1482
- by_date = _extract_daily_balance_by_date(md)
1483
- if len(by_date) < 3:
1484
- return md
1485
-
1486
- points = []
1487
- seen = set()
1488
- for k, v in by_date.items():
1489
- m = re.match(r"^\s*(\d{1,2})/(\d{1,2})(?:/\d{2,4})?\s*$", k or "")
1490
- if not m:
1491
- continue
1492
- mm = int(m.group(1))
1493
- dd = int(m.group(2))
1494
- key = (mm, dd)
1495
- if key in seen:
1496
- continue
1497
- seen.add(key)
1498
- points.append((mm, dd, v))
1499
- has_jan = any(mm == 1 for mm, _, _ in points)
1500
- # Statements often include prior-month carry-forward like 12/31 followed by Jan activity.
1501
- # Keep late-December carry-forward rows before January rows.
1502
- points.sort(key=lambda x: ((-1 if has_jan and x[0] == 12 else x[0]), x[1]))
1503
- if len(points) < 3:
1504
- return md
1505
-
1506
- first_bal = points[0][2]
1507
- last_bal = points[-1][2]
1508
- if first_bal is None or last_bal is None:
1509
- return md
1510
-
1511
- # Remove noisy posting tables but keep account summary and daily balance summary tables.
1512
- def _drop_noisy_table(m: re.Match) -> str:
1513
- t = m.group(0)
1514
- plain = _cell_plain_text(t).lower()
1515
- if "daily balance summary" in plain or "daily ending balance" in plain or "daily balance" in plain:
1516
- return t
1517
- if "account summary" in plain or "checking summary" in plain:
1518
- return t
1519
- return ""
1520
-
1521
- rows = [
1522
- "<tr><th>Date</th><th>Description</th><th>Credit</th><th>Debit</th><th>Balance</th></tr>",
1523
- f"<tr><td>{points[0][0]:02d}/{points[0][1]:02d}</td><td>BALANCE FORWARD</td><td></td><td></td><td>{_fmt_money(first_bal)}</td></tr>",
1524
- ]
1525
- for i in range(1, len(points)):
1526
- mm, dd, bal = points[i]
1527
- prev = points[i - 1][2]
1528
- delta = float(bal) - float(prev)
1529
- credit = _fmt_money(delta) if delta > 0 else ""
1530
- debit = _fmt_money(abs(delta)) if delta < 0 else ""
1531
- rows.append(
1532
- f"<tr><td>{mm:02d}/{dd:02d}</td><td>NET DAILY CHANGE</td><td>{credit}</td><td>{debit}</td><td>{_fmt_money(bal)}</td></tr>"
1533
- )
1534
- synth = "<table>\n" + "\n".join(rows) + "\n</table>"
1535
- cleaned = re.sub(r"<table\b[^>]*>.*?</table>", _drop_noisy_table, md, flags=re.IGNORECASE | re.DOTALL)
1536
- preface = (
1537
- "## Balance Snapshot\n"
1538
- f"Beginning balance: {_fmt_money(float(first_bal))}\n"
1539
- f"Ending balance: {_fmt_money(float(last_bal))}\n\n"
1540
- "The following reconciliation ledger is synthesized from daily balance points. "
1541
- "Each row is the net day-over-day movement with a deterministic running balance. "
1542
- "This representation is designed for robust downstream extraction and reconciliation.\n\n"
1543
- )
1544
- return preface + cleaned + "\n\n" + synth
1545
-
1546
-
1547
  def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
1548
  import pymupdf as fitz
1549
  from PIL import Image
@@ -1570,7 +221,6 @@ def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
1570
  canvas.paste(img, (pad_l, pad_t))
1571
  img = canvas
1572
 
1573
- # Include a per-run unique tag to avoid filename collisions across parallel local runs.
1574
  uniq = uuid.uuid4().hex[:10]
1575
  img_path = os.path.join(tempfile.gettempdir(), f"glmocr_page_{os.getpid()}_{uniq}_{i}.png")
1576
  img.save(img_path, "PNG", compress_level=PAGE_PNG_COMPRESS_LEVEL)
@@ -1644,10 +294,10 @@ def run_ocr(uploaded_file):
1644
  if not (hdr and hdr.strip()) and page_num < len(page_images):
1645
  hdr = ocr_zone(page_images[page_num], 0, he)
1646
  if hdr and hdr.strip():
1647
- parts.append(light_stabilize_markdown(fix_account_number(normalize_money_glyphs(hdr.strip()))))
1648
 
1649
  if page_md and page_md.strip():
1650
- parts.append(light_stabilize_markdown(page_md.strip()))
1651
 
1652
  if ENABLE_FOOTER_OCR and page_num < len(page_images):
1653
  ftr = ""
@@ -1658,7 +308,7 @@ def run_ocr(uploaded_file):
1658
  if not (ftr and ftr.strip()):
1659
  ftr = ocr_zone(page_images[page_num], fs, 1.0)
1660
  if ftr and ftr.strip():
1661
- ftr_clean = normalize_money_glyphs(ftr.strip())
1662
 
1663
  ftr_first_line = next(
1664
  (ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
@@ -1681,20 +331,6 @@ def run_ocr(uploaded_file):
1681
  all_pages.append("\n\n".join(parts))
1682
 
1683
  merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
1684
- if merged and merged != "(No content)" and not merged.lstrip().startswith("Error:"):
1685
- merged = stabilize_table_markup(merged, rounds=2)
1686
- merged = repair_thead_cell_semantics(merged)
1687
- merged = infer_credit_debit_from_balance_deltas(merged)
1688
- summary_forced = _force_summary_ledger_fallback(merged)
1689
- if summary_forced != merged:
1690
- merged = summary_forced
1691
- else:
1692
- merged = _force_ledger_from_daily_balances(merged)
1693
- merged = strip_ultra_long_digit_tokens(merged)
1694
- merged = mask_reference_numeric_ids(merged)
1695
- merged = mask_debit_card_auth_codes(merged)
1696
- merged = strip_glued_card_number_suffixes(merged)
1697
- merged = strip_degenerate_html_tables(merged)
1698
  return merged
1699
 
1700
  except Exception as e:
@@ -1716,17 +352,13 @@ def _create_gradio_demo():
1716
  import gradio as gr
1717
 
1718
  with gr.Blocks(title="GLM-OCR (simple)") as demo:
1719
- gr.Markdown(
1720
- "# GLM-OCR (simple)\n"
1721
- "Upload a PDF or image. Header and footer bands are included; "
1722
- "body OCR is passed through with only light markdown cleanup."
1723
- )
1724
  file_in = gr.File(
1725
  label="Upload PDF or image",
1726
  file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"],
1727
  )
1728
  run_btn = gr.Button("Run OCR", variant="primary")
1729
- out = gr.Textbox(lines=40, label="Output (markdown / light HTML)")
1730
  run_btn.click(fn=run_ocr, inputs=file_in, outputs=out)
1731
  return demo
1732
 
 
1
  #!/usr/bin/env python3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  import logging
4
  import os
5
  import re
6
  import tempfile
7
  import uuid
8
+ from typing import List, Tuple
 
9
 
10
  import yaml
11
 
 
15
  GLMOCR_BASE = os.path.dirname(glmocr.__file__)
16
  CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
17
  except ImportError:
18
+ glmocr = None
19
  GLMOCR_BASE = ""
20
  CONFIG_PATH = ""
21
 
22
  log = logging.getLogger("glmocr_simple_app")
23
  logging.basicConfig(level=logging.INFO)
24
 
 
 
 
 
 
25
  GLMOCR_API_KEY = "cee1d52dd91a4ab591b3f6e105f8ad89.LgbQTECuzX0zrito"
26
  if not GLMOCR_API_KEY:
27
  log.warning(
28
  "No ZHIPU_API_KEY or GLMOCR_API_KEY in environment; GlmOcr() will fail until you set one."
29
  )
30
 
 
 
31
  RENDER_SCALE = 3.0
 
 
 
32
  PAD_LEFT_FRAC = 0.035
33
  PAD_RIGHT_FRAC = 0.10
34
  PAD_TOP_FRAC = 0.018
35
  PAD_BOTTOM_FRAC = 0.018
36
 
 
 
 
 
 
 
37
  ENABLE_CONTRAST = True
 
38
  CONTRAST_FACTOR = 1.18
 
 
39
  ENABLE_UNSHARP = True
40
  UNSHARP_RADIUS = 0.78
41
  UNSHARP_PERCENT = 76
 
43
 
44
  DEFAULT_ZONE_FRAC = 0.12
45
  PDF_HEADER_BAND_FRAC = 0.10
 
46
  ENABLE_FOOTER_OCR = True
47
  PDF_FOOTER_BAND_FRAC = 0.88
48
 
49
  MIN_CROP_HEIGHT = 112
50
  MIN_CROP_PIXELS = 112 * 112
 
 
51
  PAGE_PNG_COMPRESS_LEVEL = 3
 
52
  ZONE_JPEG_QUALITY = 95
53
 
 
 
54
  _parser = None
55
 
56
 
57
  def _enhance_raster_for_ocr(img):
 
 
 
 
 
58
  from PIL import ImageEnhance, ImageFilter
59
 
60
  if ENABLE_CONTRAST:
 
195
  return ""
196
 
197
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
198
  def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
199
  import pymupdf as fitz
200
  from PIL import Image
 
221
  canvas.paste(img, (pad_l, pad_t))
222
  img = canvas
223
 
 
224
  uniq = uuid.uuid4().hex[:10]
225
  img_path = os.path.join(tempfile.gettempdir(), f"glmocr_page_{os.getpid()}_{uniq}_{i}.png")
226
  img.save(img_path, "PNG", compress_level=PAGE_PNG_COMPRESS_LEVEL)
 
294
  if not (hdr and hdr.strip()) and page_num < len(page_images):
295
  hdr = ocr_zone(page_images[page_num], 0, he)
296
  if hdr and hdr.strip():
297
+ parts.append(hdr.strip())
298
 
299
  if page_md and page_md.strip():
300
+ parts.append(page_md.strip())
301
 
302
  if ENABLE_FOOTER_OCR and page_num < len(page_images):
303
  ftr = ""
 
308
  if not (ftr and ftr.strip()):
309
  ftr = ocr_zone(page_images[page_num], fs, 1.0)
310
  if ftr and ftr.strip():
311
+ ftr_clean = ftr.strip()
312
 
313
  ftr_first_line = next(
314
  (ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
 
331
  all_pages.append("\n\n".join(parts))
332
 
333
  merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
334
  return merged
335
 
336
  except Exception as e:
 
352
  import gradio as gr
353
 
354
  with gr.Blocks(title="GLM-OCR (simple)") as demo:
355
+ gr.Markdown("# GLM-OCR")
 
 
 
 
356
  file_in = gr.File(
357
  label="Upload PDF or image",
358
  file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp"],
359
  )
360
  run_btn = gr.Button("Run OCR", variant="primary")
361
+ out = gr.Textbox(lines=40, label="Output (markdown / HTML from model)")
362
  run_btn.click(fn=run_ocr, inputs=file_in, outputs=out)
363
  return demo
364