Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -36,16 +36,18 @@ Primary goals for reconcile rate:
|
|
| 36 |
debit pattern) and at least one ITM DEPOSIT row exists, OCR stacked headings wrong — emit
|
| 37 |
Electronic Credits, then the table, then Electronic Debits (section continues below). Safe no-op
|
| 38 |
if the Debits heading was not before the table or any row looks like a card/ATM debit.
|
| 39 |
-
GUARD: Do NOT fire Fix 17c if ALL rows are ITM DEPOSIT — that indicates a Deposits (continued)
|
| 40 |
-
block that was incorrectly preceded by EC/ED headings, not a genuine Electronic Credits table.
|
| 41 |
- Fix 17c-html: Same as 17c using raw HTML cues when TableGridParser/header normalization would skip the table.
|
| 42 |
-
Same ITM-only guard applied.
|
| 43 |
- Fix 17d: Drop a second Electronic Debits block + DA3 table when its rows are a loose (date+amount)
|
| 44 |
multiset subset of the immediately preceding DA3 table (OCR duplicate tail / wrong split).
|
| 45 |
-
|
| 46 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
- Fix 17e: Repeat EC/ED fused + 17d passes until stable (handles chained replacements). Centered section titles
|
| 48 |
-
match both align="center" and style="text-align:center" on divs.
|
| 49 |
- Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
|
| 50 |
(later subset of earlier, earlier subset of later, or identical row sets)
|
| 51 |
- Fix 18: adjacent DA3 tables — if the last K data rows of table N match the first K rows of table N+1
|
|
@@ -92,9 +94,6 @@ Primary goals for reconcile rate:
|
|
| 92 |
earlier larger DA3 table — catches OCR-truncated description duplicates such as
|
| 93 |
the BoA service-fee page where a VIDDYOZE -1.11 fragment table appears after the
|
| 94 |
full service-fee table with a longer description string.
|
| 95 |
-
- Fix 17c/17c-html/17d GUARD (UCB page 3): Prevent misclassification of "Deposits (continued)"
|
| 96 |
-
ITM DEPOSIT blocks as Electronic Credits, and prevent Fix 17d from dropping the
|
| 97 |
-
real Electronic Credits table (POS Return / Acct Fund INST / DDA Transfer IN).
|
| 98 |
"""
|
| 99 |
|
| 100 |
# Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
|
|
@@ -1123,6 +1122,10 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
|
|
| 1123 |
key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
|
| 1124 |
tl_freq[key] = tl_freq.get(key, 0) + 1
|
| 1125 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1126 |
global _tl_freq_context
|
| 1127 |
tl_dedupe = {}
|
| 1128 |
for r in tl_rows:
|
|
@@ -2682,7 +2685,7 @@ def _transaction_row_dedupe_key(cells):
|
|
| 2682 |
date_s = re.sub(r"\s+", "", date_s)
|
| 2683 |
amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
|
| 2684 |
desc_s = html.unescape(desc_s)
|
| 2685 |
-
desc_s = desc_s.replace("`", "'").replace("
|
| 2686 |
desc_s = re.sub(r"\s+", "", desc_s.lower())
|
| 2687 |
return (date_s, desc_s, amt_s)
|
| 2688 |
|
|
@@ -2751,6 +2754,11 @@ _DA3_MONEY_TOKEN = re.compile(
|
|
| 2751 |
)
|
| 2752 |
_MSFT_PIN_G_REF = re.compile(r"MICROSOFT#(G\d{6,16})", re.IGNORECASE)
|
| 2753 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2754 |
_tl_freq_context: dict = {}
|
| 2755 |
|
| 2756 |
_DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
|
|
@@ -2841,6 +2849,12 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
|
|
| 2841 |
break
|
| 2842 |
j += 1
|
| 2843 |
run_len = j - i
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2844 |
tl_confirmed = _tl_freq_context.get(k, 0)
|
| 2845 |
cap = max(tl_confirmed, 2) if tl_confirmed > 0 else 2
|
| 2846 |
take = min(run_len, cap)
|
|
@@ -2935,38 +2949,10 @@ def _ucb_has_itm_deposit_data_row(grid) -> bool:
|
|
| 2935 |
return False
|
| 2936 |
|
| 2937 |
|
| 2938 |
-
def _ucb_all_rows_are_itm_deposit(grid) -> bool:
|
| 2939 |
-
"""
|
| 2940 |
-
Returns True if ALL data rows (row index >= 1) in a DA3 grid are
|
| 2941 |
-
ITM DEPOSIT rows. Used to guard Fix 17c / 17c-html: a table that is
|
| 2942 |
-
exclusively ITM DEPOSIT entries is a "Deposits (continued)" block,
|
| 2943 |
-
NOT an Electronic Credits table, even when it has no strong-debit rows.
|
| 2944 |
-
"""
|
| 2945 |
-
if not grid or len(grid) < 2:
|
| 2946 |
-
return False
|
| 2947 |
-
data_rows = [grid[r] for r in range(1, len(grid))
|
| 2948 |
-
if any(str(c or "").strip() for c in grid[r])]
|
| 2949 |
-
if not data_rows:
|
| 2950 |
-
return False
|
| 2951 |
-
return all(_is_itm_deposit_row_ucb(grid, r) for r in range(1, len(grid))
|
| 2952 |
-
if any(str(c or "").strip() for c in grid[r]))
|
| 2953 |
-
|
| 2954 |
-
|
| 2955 |
def _ucb_table_credits_only_no_strong_debits(grid) -> bool:
|
| 2956 |
-
"""
|
| 2957 |
-
True if DA3 table has real data rows but none match strong
|
| 2958 |
-
electronic-debit patterns AND at least one ITM DEPOSIT row exists.
|
| 2959 |
-
|
| 2960 |
-
FIX 17c GUARD: Returns False when ALL data rows are ITM DEPOSIT.
|
| 2961 |
-
That case is a "Deposits (continued)" block — not Electronic Credits —
|
| 2962 |
-
so Fix 17c must NOT reorder it under an EC heading.
|
| 2963 |
-
"""
|
| 2964 |
if not grid or len(grid) < 2 or not _is_da3_header(grid):
|
| 2965 |
return False
|
| 2966 |
-
# Guard: if every row is ITM DEPOSIT, this is Deposits (continued),
|
| 2967 |
-
# not Electronic Credits. Do not trigger Fix 17c.
|
| 2968 |
-
if _ucb_all_rows_are_itm_deposit(grid):
|
| 2969 |
-
return False
|
| 2970 |
has_data = False
|
| 2971 |
for r in range(1, len(grid)):
|
| 2972 |
row_txt = _row_text_full_ucb(grid, r).strip()
|
|
@@ -2982,9 +2968,6 @@ def _ucb_html_table_body_looks_itm_credits_only(table_html: str) -> bool:
|
|
| 2982 |
"""
|
| 2983 |
Detect ITM-only credit block without relying on parsed grid headers (Fix 17c-html).
|
| 2984 |
Conservative: require ITM deposit text and absence of common electronic-debit verbs in the HTML.
|
| 2985 |
-
|
| 2986 |
-
FIX 17c-html GUARD: Returns False when the table contains ONLY ITM DEPOSIT rows
|
| 2987 |
-
(no other description text), as that indicates a Deposits (continued) block.
|
| 2988 |
"""
|
| 2989 |
if not table_html or "<table" not in table_html.lower():
|
| 2990 |
return False
|
|
@@ -2995,30 +2978,6 @@ def _ucb_html_table_body_looks_itm_credits_only(table_html: str) -> bool:
|
|
| 2995 |
return False
|
| 2996 |
if "recur payment" in low or "debit card" in low:
|
| 2997 |
return False
|
| 2998 |
-
# Additional guard: if every <td> description cell only contains "ITM DEPOSIT"
|
| 2999 |
-
# (no other transaction types), treat this as Deposits (continued), not EC.
|
| 3000 |
-
# Extract description cell texts from <td> elements (skip header row <th>).
|
| 3001 |
-
td_texts = re.findall(r"<td>([^<]*)</td>", table_html, re.IGNORECASE)
|
| 3002 |
-
if td_texts:
|
| 3003 |
-
# td_texts from a DA3 table come in triplets: date, desc, amount
|
| 3004 |
-
# Check if any non-date, non-amount cell has text other than "ITM DEPOSIT"
|
| 3005 |
-
non_itm_desc = False
|
| 3006 |
-
for i, txt in enumerate(td_texts):
|
| 3007 |
-
txt_stripped = txt.strip().lower()
|
| 3008 |
-
if not txt_stripped:
|
| 3009 |
-
continue
|
| 3010 |
-
# Skip date-like tokens and amount-like tokens
|
| 3011 |
-
if re.match(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$", txt_stripped):
|
| 3012 |
-
continue
|
| 3013 |
-
if re.match(r"^\$?[\d,]+\.\d{2}$", txt_stripped):
|
| 3014 |
-
continue
|
| 3015 |
-
# It's a description cell
|
| 3016 |
-
if "itm deposit" not in txt_stripped and "itm dep" not in txt_stripped:
|
| 3017 |
-
non_itm_desc = True
|
| 3018 |
-
break
|
| 3019 |
-
if not non_itm_desc:
|
| 3020 |
-
# All description cells are ITM DEPOSIT — this is Deposits (continued)
|
| 3021 |
-
return False
|
| 3022 |
return True
|
| 3023 |
|
| 3024 |
|
|
@@ -3076,7 +3035,6 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
|
|
| 3076 |
|
| 3077 |
# Fix 17c: headings stacked as EC / ED / table but rows are all credits (e.g. ITM-only) —
|
| 3078 |
# put the table under Electronic Credits, then leave Electronic Debits for continuation.
|
| 3079 |
-
# GUARD: Do NOT fire if all rows are ITM DEPOSIT (that is Deposits continued, not EC).
|
| 3080 |
if had_ed_heading_immediately_before_table and _ucb_table_credits_only_no_strong_debits(grid):
|
| 3081 |
sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
|
| 3082 |
parts = []
|
|
@@ -3092,7 +3050,6 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
|
|
| 3092 |
return "\n\n".join(parts)
|
| 3093 |
|
| 3094 |
# Fix 17c-html: same reorder when grid header checks fail but raw HTML is clearly ITM-only credits.
|
| 3095 |
-
# GUARD: _ucb_html_table_body_looks_itm_credits_only now returns False for pure ITM DEPOSIT tables.
|
| 3096 |
if had_ed_heading_immediately_before_table and _ucb_html_table_body_looks_itm_credits_only(table_html):
|
| 3097 |
sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
|
| 3098 |
parts = []
|
|
@@ -3119,6 +3076,8 @@ _UCB_CENTER_DIV = (
|
|
| 3119 |
r')'
|
| 3120 |
)
|
| 3121 |
|
|
|
|
|
|
|
| 3122 |
_UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
|
| 3123 |
r'(?P<lead>(?:' + _UCB_CENTER_DIV + r'\s*</div>\s*)?)'
|
| 3124 |
r'(?P<ecdiv>' + _UCB_CENTER_DIV + r'\s*Electronic\s+Credits\s*</div>\s*)'
|
|
@@ -3128,32 +3087,12 @@ _UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
|
|
| 3128 |
)
|
| 3129 |
|
| 3130 |
|
| 3131 |
-
def _ucb_table_html_has_genuine_credit_rows(table_html: str) -> bool:
|
| 3132 |
-
"""
|
| 3133 |
-
Returns True if the table contains rows that are clearly credit-side
|
| 3134 |
-
(POS Return, Acct Fund INST/CUPERTINO, DDA Transfer IN).
|
| 3135 |
-
Used by Fix 17d guard to prevent dropping the real EC table.
|
| 3136 |
-
"""
|
| 3137 |
-
if not table_html:
|
| 3138 |
-
return False
|
| 3139 |
-
low = table_html.lower()
|
| 3140 |
-
if "pos return" in low:
|
| 3141 |
-
return True
|
| 3142 |
-
if "acct fund" in low and ("inst" in low or "cupertino" in low):
|
| 3143 |
-
return True
|
| 3144 |
-
if "dda transfer" in low and "in" in low:
|
| 3145 |
-
return True
|
| 3146 |
-
return False
|
| 3147 |
-
|
| 3148 |
-
|
| 3149 |
def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
|
| 3150 |
"""
|
| 3151 |
-
Fix 17d: Remove a duplicate <div>Electronic Debits</div> + <table> when that table's rows are a
|
| 3152 |
loose (date, amount) multiset subset of the DA3 table that appears immediately before this block.
|
| 3153 |
-
|
| 3154 |
-
|
| 3155 |
-
Acct Fund INST/CUPERTINO, DDA Transfer IN). Those rows belong under Electronic Credits
|
| 3156 |
-
and must not be silently discarded as ED duplicates.
|
| 3157 |
"""
|
| 3158 |
if not text or "electronic debits" not in text.lower():
|
| 3159 |
return text
|
|
@@ -3165,6 +3104,7 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
|
|
| 3165 |
)
|
| 3166 |
tbl_pat = re.compile(r'<table[^>]*>.*?</table>', re.DOTALL | re.IGNORECASE)
|
| 3167 |
_DATE_NONEMPTY = re.compile(r"^\d{1,2}[/\-]\d{2}")
|
|
|
|
| 3168 |
_AMT_STRIPPED_OK = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
|
| 3169 |
|
| 3170 |
def _loose_da_key_cells(cells):
|
|
@@ -3193,6 +3133,35 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
|
|
| 3193 |
keys.append(k)
|
| 3194 |
return keys
|
| 3195 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3196 |
for _ in range(12):
|
| 3197 |
changed = False
|
| 3198 |
for m in ed_block.finditer(text):
|
|
@@ -3206,29 +3175,156 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
|
|
| 3206 |
g_frag = _parse_da3_grid_from_table_html(m.group("tbl"))
|
| 3207 |
kb = _loose_keys_from_grid(g_big)
|
| 3208 |
kf = _loose_keys_from_grid(g_frag)
|
| 3209 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3210 |
continue
|
| 3211 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3212 |
continue
|
| 3213 |
-
|
| 3214 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3215 |
continue
|
| 3216 |
-
|
| 3217 |
-
# These rows belong under Electronic Credits, not as ED duplicates.
|
| 3218 |
-
if _ucb_table_html_has_genuine_credit_rows(m.group("tbl")):
|
| 3219 |
-
log.info(
|
| 3220 |
-
"Fix 17d: skipping drop of table with genuine credit rows "
|
| 3221 |
-
"(POS Return / Acct Fund INST / DDA Transfer IN)"
|
| 3222 |
-
)
|
| 3223 |
continue
|
| 3224 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3225 |
changed = True
|
| 3226 |
break
|
| 3227 |
if not changed:
|
| 3228 |
break
|
| 3229 |
return text
|
| 3230 |
except Exception as e:
|
| 3231 |
-
log.warning("
|
| 3232 |
return text
|
| 3233 |
|
| 3234 |
|
|
@@ -3290,10 +3386,11 @@ def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
|
|
| 3290 |
return new
|
| 3291 |
|
| 3292 |
text = _UCB_DEP_CONT_BLOCK.sub(_repl, text)
|
| 3293 |
-
# Fix 17e: repeat fused EC/ED fix + duplicate ED strip
|
| 3294 |
for _ in range(16):
|
| 3295 |
nxt = _split_ucb_fused_ec_ed_headings_single_table_html(text)
|
| 3296 |
nxt = _ucb_strip_redundant_ed_subset_fragment(nxt)
|
|
|
|
| 3297 |
if nxt == text:
|
| 3298 |
break
|
| 3299 |
text = nxt
|
|
@@ -3505,6 +3602,21 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
|
|
| 3505 |
"""
|
| 3506 |
Fix 30: Drop small DA3 table fragments (1-5 data rows) that are a loose
|
| 3507 |
subset of an earlier larger DA3 table, matched by (date, amount) key alone.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3508 |
"""
|
| 3509 |
if not text or "<table" not in text.lower():
|
| 3510 |
return text
|
|
@@ -3518,6 +3630,7 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
|
|
| 3518 |
_AMT_NONEMPTY = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
|
| 3519 |
|
| 3520 |
def _da_key(cells):
|
|
|
|
| 3521 |
parts = [str(c or "").strip() for c in cells[:3]]
|
| 3522 |
while len(parts) < 3:
|
| 3523 |
parts.append("")
|
|
@@ -3526,6 +3639,7 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
|
|
| 3526 |
amt_s = amt_s.replace("\u2212", "-").replace("\u2013", "-")
|
| 3527 |
return (date_s, amt_s)
|
| 3528 |
|
|
|
|
| 3529 |
da3_info = []
|
| 3530 |
for m in matches:
|
| 3531 |
g = _parse_da3_grid_from_table_html(m.group(0))
|
|
@@ -3559,6 +3673,8 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
|
|
| 3559 |
if tj["n"] <= ti["n"]:
|
| 3560 |
continue
|
| 3561 |
big_keys = Counter(tj["keys"])
|
|
|
|
|
|
|
| 3562 |
if all(frag_keys[k] <= big_keys.get(k, 0) for k in frag_keys):
|
| 3563 |
drop_set.add(ti["match"].start())
|
| 3564 |
log.info(
|
|
@@ -3859,6 +3975,14 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
|
|
| 3859 |
"""
|
| 3860 |
Fix 28: When OCR prepends the rows of a preceding credits/deposits DA3 table
|
| 3861 |
onto the top of a debits/withdrawals DA3 table, strip that prefix.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3862 |
"""
|
| 3863 |
if not text or "<table" not in text.lower():
|
| 3864 |
return text
|
|
@@ -4050,6 +4174,8 @@ def stabilize_tables_and_text(page_md: str) -> str:
|
|
| 4050 |
if not page_md:
|
| 4051 |
return page_md
|
| 4052 |
|
|
|
|
|
|
|
| 4053 |
_tl_freq_context.clear()
|
| 4054 |
|
| 4055 |
page_md = normalize_money_glyphs(page_md)
|
|
@@ -4167,6 +4293,7 @@ def run_ocr(uploaded_file):
|
|
| 4167 |
|
| 4168 |
all_pages = []
|
| 4169 |
for page_num, page_result in enumerate(results):
|
|
|
|
| 4170 |
_tl_freq_context.clear()
|
| 4171 |
page_md, regions = get_page_md_and_regions(page_result)
|
| 4172 |
img_h = page_heights[page_num] if page_num < len(page_heights) else 1000
|
|
|
|
| 36 |
debit pattern) and at least one ITM DEPOSIT row exists, OCR stacked headings wrong — emit
|
| 37 |
Electronic Credits, then the table, then Electronic Debits (section continues below). Safe no-op
|
| 38 |
if the Debits heading was not before the table or any row looks like a card/ATM debit.
|
|
|
|
|
|
|
| 39 |
- Fix 17c-html: Same as 17c using raw HTML cues when TableGridParser/header normalization would skip the table.
|
|
|
|
| 40 |
- Fix 17d: Drop a second Electronic Debits block + DA3 table when its rows are a loose (date+amount)
|
| 41 |
multiset subset of the immediately preceding DA3 table (OCR duplicate tail / wrong split).
|
| 42 |
+
- Fix 17f: Same as 17d when the duplicate table *starts with* ITM DEPOSIT rows (copied from Electronic Credits)
|
| 43 |
+
then repeats debits from the table above — 17d alone misses those because ITM keys are not in the
|
| 44 |
+
debits-only predecessor table. Strip ITM rows from the multiset check, then apply the 17d subset rule.
|
| 45 |
+
- Fix 17g: When 17f cannot drop the whole block (e.g. one orphan debit row only appears in the duplicate tail),
|
| 46 |
+
drop leading ITM rows and each fragment row whose (date, amount) still has count in the previous
|
| 47 |
+
debits table; remove the ED wrapper if the table becomes empty. If a small orphan tail remains
|
| 48 |
+
(≤6 rows), append those rows to the preceding debits table and drop the extra ED heading.
|
| 49 |
- Fix 17e: Repeat EC/ED fused + 17d passes until stable (handles chained replacements). Centered section titles
|
| 50 |
+
match both align=\"center\" and style=\"text-align:center\" on divs.
|
| 51 |
- Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
|
| 52 |
(later subset of earlier, earlier subset of later, or identical row sets)
|
| 53 |
- Fix 18: adjacent DA3 tables — if the last K data rows of table N match the first K rows of table N+1
|
|
|
|
| 94 |
earlier larger DA3 table — catches OCR-truncated description duplicates such as
|
| 95 |
the BoA service-fee page where a VIDDYOZE -1.11 fragment table appears after the
|
| 96 |
full service-fee table with a longer description string.
|
|
|
|
|
|
|
|
|
|
| 97 |
"""
|
| 98 |
|
| 99 |
# Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
|
|
|
|
| 1122 |
key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
|
| 1123 |
tl_freq[key] = tl_freq.get(key, 0) + 1
|
| 1124 |
|
| 1125 |
+
# Share with _dedupe_duplicate_rows_in_da3_table (Fix-20 override).
|
| 1126 |
+
# Keys MUST match _transaction_row_dedupe_key (date/desc/amount, no spaces in
|
| 1127 |
+
# date/desc; amount stripped of $/,/space). The injection map tl_freq uses
|
| 1128 |
+
# _norm() with spaces — different tuple shape — so we keep a parallel count.
|
| 1129 |
global _tl_freq_context
|
| 1130 |
tl_dedupe = {}
|
| 1131 |
for r in tl_rows:
|
|
|
|
| 2685 |
date_s = re.sub(r"\s+", "", date_s)
|
| 2686 |
amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
|
| 2687 |
desc_s = html.unescape(desc_s)
|
| 2688 |
+
desc_s = desc_s.replace("`", "'").replace("'", "'")
|
| 2689 |
desc_s = re.sub(r"\s+", "", desc_s.lower())
|
| 2690 |
return (date_s, desc_s, amt_s)
|
| 2691 |
|
|
|
|
| 2754 |
)
|
| 2755 |
_MSFT_PIN_G_REF = re.compile(r"MICROSOFT#(G\d{6,16})", re.IGNORECASE)
|
| 2756 |
|
| 2757 |
+
# Module-level text-layer frequency context (Fix-20 override).
|
| 2758 |
+
# Set by _patch_ocr_with_textlayer; consumed by normalize_html_tables in run_ocr's
|
| 2759 |
+
# post-patch pass. Cleared at the start of stabilize_tables_and_text (new page)
|
| 2760 |
+
# and on patch failure so keys never bleed across pages.
|
| 2761 |
+
# Keys match _transaction_row_dedupe_key (not the _norm()-based tl_freq map).
|
| 2762 |
_tl_freq_context: dict = {}
|
| 2763 |
|
| 2764 |
_DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
|
|
|
|
| 2849 |
break
|
| 2850 |
j += 1
|
| 2851 |
run_len = j - i
|
| 2852 |
+
# Cap at 2 by default (Fix 20: collapse OCR stutters) but honour
|
| 2853 |
+
# the text-layer-confirmed count when available, so legitimate
|
| 2854 |
+
# repeated bank transactions (e.g. 6x Temporary Credit Reversal
|
| 2855 |
+
# on the same date/amount) are not incorrectly collapsed to 2.
|
| 2856 |
+
# Keys absent from _tl_freq_context still cap at 2 — safe no-op
|
| 2857 |
+
# for scanned PDFs or pages where no text layer was extracted.
|
| 2858 |
tl_confirmed = _tl_freq_context.get(k, 0)
|
| 2859 |
cap = max(tl_confirmed, 2) if tl_confirmed > 0 else 2
|
| 2860 |
take = min(run_len, cap)
|
|
|
|
| 2949 |
return False
|
| 2950 |
|
| 2951 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2952 |
def _ucb_table_credits_only_no_strong_debits(grid) -> bool:
|
| 2953 |
+
"""True if DA3 table has real data rows but none match strong electronic-debit patterns."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2954 |
if not grid or len(grid) < 2 or not _is_da3_header(grid):
|
| 2955 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2956 |
has_data = False
|
| 2957 |
for r in range(1, len(grid)):
|
| 2958 |
row_txt = _row_text_full_ucb(grid, r).strip()
|
|
|
|
| 2968 |
"""
|
| 2969 |
Detect ITM-only credit block without relying on parsed grid headers (Fix 17c-html).
|
| 2970 |
Conservative: require ITM deposit text and absence of common electronic-debit verbs in the HTML.
|
|
|
|
|
|
|
|
|
|
| 2971 |
"""
|
| 2972 |
if not table_html or "<table" not in table_html.lower():
|
| 2973 |
return False
|
|
|
|
| 2978 |
return False
|
| 2979 |
if "recur payment" in low or "debit card" in low:
|
| 2980 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2981 |
return True
|
| 2982 |
|
| 2983 |
|
|
|
|
| 3035 |
|
| 3036 |
# Fix 17c: headings stacked as EC / ED / table but rows are all credits (e.g. ITM-only) —
|
| 3037 |
# put the table under Electronic Credits, then leave Electronic Debits for continuation.
|
|
|
|
| 3038 |
if had_ed_heading_immediately_before_table and _ucb_table_credits_only_no_strong_debits(grid):
|
| 3039 |
sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
|
| 3040 |
parts = []
|
|
|
|
| 3050 |
return "\n\n".join(parts)
|
| 3051 |
|
| 3052 |
# Fix 17c-html: same reorder when grid header checks fail but raw HTML is clearly ITM-only credits.
|
|
|
|
| 3053 |
if had_ed_heading_immediately_before_table and _ucb_html_table_body_looks_itm_credits_only(table_html):
|
| 3054 |
sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
|
| 3055 |
parts = []
|
|
|
|
| 3076 |
r')'
|
| 3077 |
)
|
| 3078 |
|
| 3079 |
+
# Optional empty centered banner only; EC/ED title divs are re-emitted after a successful split
|
| 3080 |
+
# so we do not duplicate headings.
|
| 3081 |
_UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
|
| 3082 |
r'(?P<lead>(?:' + _UCB_CENTER_DIV + r'\s*</div>\s*)?)'
|
| 3083 |
r'(?P<ecdiv>' + _UCB_CENTER_DIV + r'\s*Electronic\s+Credits\s*</div>\s*)'
|
|
|
|
| 3087 |
)
|
| 3088 |
|
| 3089 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3090 |
def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
|
| 3091 |
"""
|
| 3092 |
+
Fix 17d / 17f: Remove a duplicate <div>Electronic Debits</div> + <table> when that table's rows are a
|
| 3093 |
loose (date, amount) multiset subset of the DA3 table that appears immediately before this block.
|
| 3094 |
+
Fix 17f: If the fragment also contains ITM DEPOSIT rows (mis-copied under Debits), ignore those rows
|
| 3095 |
+
for the subset check so the tail still compares against the preceding debits table.
|
|
|
|
|
|
|
| 3096 |
"""
|
| 3097 |
if not text or "electronic debits" not in text.lower():
|
| 3098 |
return text
|
|
|
|
| 3104 |
)
|
| 3105 |
tbl_pat = re.compile(r'<table[^>]*>.*?</table>', re.DOTALL | re.IGNORECASE)
|
| 3106 |
_DATE_NONEMPTY = re.compile(r"^\d{1,2}[/\-]\d{2}")
|
| 3107 |
+
# Stripped amounts may be "2194.19" (no comma) — must not use comma-group regex.
|
| 3108 |
_AMT_STRIPPED_OK = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
|
| 3109 |
|
| 3110 |
def _loose_da_key_cells(cells):
|
|
|
|
| 3133 |
keys.append(k)
|
| 3134 |
return keys
|
| 3135 |
|
| 3136 |
+
def _loose_keys_from_grid_skip_itm(g):
|
| 3137 |
+
if not g or not _is_da3_header(g):
|
| 3138 |
+
return None
|
| 3139 |
+
keys = []
|
| 3140 |
+
for r in range(1, len(g)):
|
| 3141 |
+
if _is_itm_deposit_row_ucb(g, r):
|
| 3142 |
+
continue
|
| 3143 |
+
cells = [str(c or "").strip() for c in g[r]]
|
| 3144 |
+
if not any(cells):
|
| 3145 |
+
continue
|
| 3146 |
+
k = _loose_da_key_cells(cells)
|
| 3147 |
+
if (
|
| 3148 |
+
_DATE_NONEMPTY.match(k[0])
|
| 3149 |
+
and _AMT_STRIPPED_OK.match(k[1])
|
| 3150 |
+
and "." in k[1]
|
| 3151 |
+
):
|
| 3152 |
+
keys.append(k)
|
| 3153 |
+
return keys
|
| 3154 |
+
|
| 3155 |
+
def _frag_has_itm_data_row(g):
|
| 3156 |
+
if not g:
|
| 3157 |
+
return False
|
| 3158 |
+
for r in range(1, len(g)):
|
| 3159 |
+
if not any(str(c or "").strip() for c in g[r]):
|
| 3160 |
+
continue
|
| 3161 |
+
if _is_itm_deposit_row_ucb(g, r):
|
| 3162 |
+
return True
|
| 3163 |
+
return False
|
| 3164 |
+
|
| 3165 |
for _ in range(12):
|
| 3166 |
changed = False
|
| 3167 |
for m in ed_block.finditer(text):
|
|
|
|
| 3175 |
g_frag = _parse_da3_grid_from_table_html(m.group("tbl"))
|
| 3176 |
kb = _loose_keys_from_grid(g_big)
|
| 3177 |
kf = _loose_keys_from_grid(g_frag)
|
| 3178 |
+
strip = False
|
| 3179 |
+
if kb and kf and len(kf) <= 24 and len(kf) < len(kb):
|
| 3180 |
+
cb, cf = Counter(kb), Counter(kf)
|
| 3181 |
+
if all(cf[k] <= cb.get(k, 0) for k in cf):
|
| 3182 |
+
strip = True
|
| 3183 |
+
# Fix 17f: ITM rows duplicated from Credits; debit tail ⊆ previous debits table
|
| 3184 |
+
if (
|
| 3185 |
+
not strip
|
| 3186 |
+
and kb
|
| 3187 |
+
and g_frag
|
| 3188 |
+
and _frag_has_itm_data_row(g_frag)
|
| 3189 |
+
):
|
| 3190 |
+
kf2 = _loose_keys_from_grid_skip_itm(g_frag)
|
| 3191 |
+
if (
|
| 3192 |
+
kf2
|
| 3193 |
+
and len(kf2) <= 24
|
| 3194 |
+
and len(kf2) < len(kb)
|
| 3195 |
+
):
|
| 3196 |
+
cb, cf2 = Counter(kb), Counter(kf2)
|
| 3197 |
+
if all(cf2[k] <= cb.get(k, 0) for k in cf2):
|
| 3198 |
+
strip = True
|
| 3199 |
+
if strip:
|
| 3200 |
+
text = text[: m.start()] + text[m.end() :]
|
| 3201 |
+
changed = True
|
| 3202 |
+
break
|
| 3203 |
+
if not changed:
|
| 3204 |
+
break
|
| 3205 |
+
return text
|
| 3206 |
+
except Exception as e:
|
| 3207 |
+
log.warning("_ucb_strip_redundant_ed_subset_fragment failed: %s", e)
|
| 3208 |
+
return text
|
| 3209 |
+
|
| 3210 |
+
|
| 3211 |
+
def _ucb_prune_ed_fragment_dupes_vs_prev_table(text: str) -> str:
|
| 3212 |
+
"""
|
| 3213 |
+
Fix 17g: Under a second Electronic Debits + DA3 table, drop leading ITM rows and rows that duplicate
|
| 3214 |
+
the immediately preceding debits table by (date, amount). Keeps orphan rows (same key absent from prev).
|
| 3215 |
+
Removes the whole block if only the header remains. If ≤6 orphan rows remain, merge them into the
|
| 3216 |
+
preceding table and remove this ED block.
|
| 3217 |
+
"""
|
| 3218 |
+
if not text or "electronic debits" not in text.lower():
|
| 3219 |
+
return text
|
| 3220 |
+
try:
|
| 3221 |
+
ed_block = re.compile(
|
| 3222 |
+
_UCB_CENTER_DIV + r"\s*Electronic\s+Debits\s*</div>\s*"
|
| 3223 |
+
r"(?P<tbl><table[^>]*>.*?</table>)",
|
| 3224 |
+
re.DOTALL | re.IGNORECASE,
|
| 3225 |
+
)
|
| 3226 |
+
tbl_pat = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
|
| 3227 |
+
_dn = re.compile(r"^\d{1,2}[/\-]\d{2}")
|
| 3228 |
+
_amt_ok = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
|
| 3229 |
+
|
| 3230 |
+
def _key_cells(cells):
|
| 3231 |
+
parts = [str(c or "").strip() for c in cells[:3]]
|
| 3232 |
+
while len(parts) < 3:
|
| 3233 |
+
parts.append("")
|
| 3234 |
+
date_s = re.sub(r"\s+", "", parts[0])
|
| 3235 |
+
amt_s = re.sub(r"[\s$,]", "", parts[2]).lower()
|
| 3236 |
+
amt_s = amt_s.replace("\u2212", "-").replace("\u2013", "-")
|
| 3237 |
+
return date_s, amt_s
|
| 3238 |
+
|
| 3239 |
+
def _row_loose_key(grid, r):
|
| 3240 |
+
cells = [str(c or "").strip() for c in grid[r]]
|
| 3241 |
+
if not any(cells):
|
| 3242 |
+
return None
|
| 3243 |
+
k = _key_cells(cells)
|
| 3244 |
+
if not (_dn.match(k[0]) and _amt_ok.match(k[1]) and "." in k[1]):
|
| 3245 |
+
return None
|
| 3246 |
+
return k
|
| 3247 |
+
|
| 3248 |
+
def _has_itm_data(grid):
|
| 3249 |
+
for r in range(1, len(grid)):
|
| 3250 |
+
if not any(str(c or "").strip() for c in grid[r]):
|
| 3251 |
continue
|
| 3252 |
+
if _is_itm_deposit_row_ucb(grid, r):
|
| 3253 |
+
return True
|
| 3254 |
+
return False
|
| 3255 |
+
|
| 3256 |
+
for _ in range(12):
|
| 3257 |
+
changed = False
|
| 3258 |
+
for m in ed_block.finditer(text):
|
| 3259 |
+
before = text[: m.start()]
|
| 3260 |
+
prev_tbl_m = None
|
| 3261 |
+
for tm in tbl_pat.finditer(before):
|
| 3262 |
+
prev_tbl_m = tm
|
| 3263 |
+
if prev_tbl_m is None:
|
| 3264 |
continue
|
| 3265 |
+
g_big = _parse_da3_grid_from_table_html(prev_tbl_m.group(0))
|
| 3266 |
+
g_frag = _parse_da3_grid_from_table_html(m.group("tbl"))
|
| 3267 |
+
if (
|
| 3268 |
+
not g_big
|
| 3269 |
+
or not g_frag
|
| 3270 |
+
or not _is_da3_header(g_big)
|
| 3271 |
+
or not _is_da3_header(g_frag)
|
| 3272 |
+
):
|
| 3273 |
continue
|
| 3274 |
+
if not _has_itm_data(g_frag):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3275 |
continue
|
| 3276 |
+
cb = Counter()
|
| 3277 |
+
for r in range(1, len(g_big)):
|
| 3278 |
+
kk = _row_loose_key(g_big, r)
|
| 3279 |
+
if kk:
|
| 3280 |
+
cb[kk] += 1
|
| 3281 |
+
new_rows = []
|
| 3282 |
+
seen_non_itm = False
|
| 3283 |
+
for r in range(1, len(g_frag)):
|
| 3284 |
+
if not any(str(c or "").strip() for c in g_frag[r]):
|
| 3285 |
+
continue
|
| 3286 |
+
if _is_itm_deposit_row_ucb(g_frag, r):
|
| 3287 |
+
if not seen_non_itm:
|
| 3288 |
+
continue
|
| 3289 |
+
continue
|
| 3290 |
+
seen_non_itm = True
|
| 3291 |
+
kk = _row_loose_key(g_frag, r)
|
| 3292 |
+
if kk and cb.get(kk, 0) > 0:
|
| 3293 |
+
cb[kk] -= 1
|
| 3294 |
+
continue
|
| 3295 |
+
new_rows.append(list(g_frag[r]))
|
| 3296 |
+
if not new_rows:
|
| 3297 |
+
text = text[: m.start()] + text[m.end() :]
|
| 3298 |
+
changed = True
|
| 3299 |
+
break
|
| 3300 |
+
if len(new_rows) <= 6:
|
| 3301 |
+
merged = [list(g_big[0])] + [list(r) for r in g_big[1:]] + new_rows
|
| 3302 |
+
max_c = max(len(r) for r in merged)
|
| 3303 |
+
for row in merged:
|
| 3304 |
+
while len(row) < max_c:
|
| 3305 |
+
row.append("")
|
| 3306 |
+
merged_html = _grid_to_html(merged)
|
| 3307 |
+
ps, pe = prev_tbl_m.start(), prev_tbl_m.end()
|
| 3308 |
+
text = text[:ps] + merged_html + text[pe : m.start()] + text[m.end() :]
|
| 3309 |
+
changed = True
|
| 3310 |
+
break
|
| 3311 |
+
hdr = [list(g_frag[0])]
|
| 3312 |
+
max_c = max(len(hdr[0]), max((len(x) for x in new_rows), default=0))
|
| 3313 |
+
for row in hdr + new_rows:
|
| 3314 |
+
while len(row) < max_c:
|
| 3315 |
+
row.append("")
|
| 3316 |
+
new_tbl = _grid_to_html(hdr + new_rows)
|
| 3317 |
+
full = m.group(0)
|
| 3318 |
+
tbl = m.group("tbl")
|
| 3319 |
+
prefix = full[: full.index(tbl)]
|
| 3320 |
+
text = text[: m.start()] + prefix + new_tbl + text[m.end() :]
|
| 3321 |
changed = True
|
| 3322 |
break
|
| 3323 |
if not changed:
|
| 3324 |
break
|
| 3325 |
return text
|
| 3326 |
except Exception as e:
|
| 3327 |
+
log.warning("_ucb_prune_ed_fragment_dupes_vs_prev_table failed: %s", e)
|
| 3328 |
return text
|
| 3329 |
|
| 3330 |
|
|
|
|
| 3386 |
return new
|
| 3387 |
|
| 3388 |
text = _UCB_DEP_CONT_BLOCK.sub(_repl, text)
|
| 3389 |
+
# Fix 17e: repeat fused EC/ED fix + duplicate ED strip + 17g prune until stable.
|
| 3390 |
for _ in range(16):
|
| 3391 |
nxt = _split_ucb_fused_ec_ed_headings_single_table_html(text)
|
| 3392 |
nxt = _ucb_strip_redundant_ed_subset_fragment(nxt)
|
| 3393 |
+
nxt = _ucb_prune_ed_fragment_dupes_vs_prev_table(nxt)
|
| 3394 |
if nxt == text:
|
| 3395 |
break
|
| 3396 |
text = nxt
|
|
|
|
| 3602 |
"""
|
| 3603 |
Fix 30: Drop small DA3 table fragments (1-5 data rows) that are a loose
|
| 3604 |
subset of an earlier larger DA3 table, matched by (date, amount) key alone.
|
| 3605 |
+
|
| 3606 |
+
This catches OCR-truncated duplicate rows where description text differs
|
| 3607 |
+
between the fragment and the anchor table, preventing the exact-key dedup
|
| 3608 |
+
in _dedupe_subset_transaction_tables_html from firing.
|
| 3609 |
+
|
| 3610 |
+
Example: BoA service-fee page — VIDDYOZE -1.11 appears in a 1-row
|
| 3611 |
+
fragment table after a full 2-row service-fee table that has the same
|
| 3612 |
+
row with a much longer description string.
|
| 3613 |
+
|
| 3614 |
+
Guards:
|
| 3615 |
+
- Fragment must have 1-5 data rows.
|
| 3616 |
+
- Anchor table must have more data rows than the fragment.
|
| 3617 |
+
- Every (date, amount) pair in the fragment must be covered by the anchor.
|
| 3618 |
+
- Date must be non-empty; amount must look like currency.
|
| 3619 |
+
- Wrapped in try/except (safe no-op on any error).
|
| 3620 |
"""
|
| 3621 |
if not text or "<table" not in text.lower():
|
| 3622 |
return text
|
|
|
|
| 3630 |
_AMT_NONEMPTY = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
|
| 3631 |
|
| 3632 |
def _da_key(cells):
|
| 3633 |
+
"""Loose (date_norm, amount_norm) key — description ignored."""
|
| 3634 |
parts = [str(c or "").strip() for c in cells[:3]]
|
| 3635 |
while len(parts) < 3:
|
| 3636 |
parts.append("")
|
|
|
|
| 3639 |
amt_s = amt_s.replace("\u2212", "-").replace("\u2013", "-")
|
| 3640 |
return (date_s, amt_s)
|
| 3641 |
|
| 3642 |
+
# Parse all DA3 tables and collect their (date, amount) key lists.
|
| 3643 |
da3_info = []
|
| 3644 |
for m in matches:
|
| 3645 |
g = _parse_da3_grid_from_table_html(m.group(0))
|
|
|
|
| 3673 |
if tj["n"] <= ti["n"]:
|
| 3674 |
continue
|
| 3675 |
big_keys = Counter(tj["keys"])
|
| 3676 |
+
# Every (date, amount) pair in the fragment must be covered
|
| 3677 |
+
# by the anchor table (multiset subset check).
|
| 3678 |
if all(frag_keys[k] <= big_keys.get(k, 0) for k in frag_keys):
|
| 3679 |
drop_set.add(ti["match"].start())
|
| 3680 |
log.info(
|
|
|
|
| 3975 |
"""
|
| 3976 |
Fix 28: When OCR prepends the rows of a preceding credits/deposits DA3 table
|
| 3977 |
onto the top of a debits/withdrawals DA3 table, strip that prefix.
|
| 3978 |
+
Safety guards (ALL must pass):
|
| 3979 |
+
1. Both tables must be valid DA3 (Date|Description|Amount header).
|
| 3980 |
+
2. Target (later) table must follow a debit/withdrawal heading.
|
| 3981 |
+
3. Source (earlier) table must follow a credit/deposit heading.
|
| 3982 |
+
4. Only the contiguous leading prefix of matching rows is removed.
|
| 3983 |
+
5. At least 1 data row must remain in the target after stripping.
|
| 3984 |
+
6. Strip count must be >= 1.
|
| 3985 |
+
7. Entire function wrapped in try/except — returns input unchanged on error.
|
| 3986 |
"""
|
| 3987 |
if not text or "<table" not in text.lower():
|
| 3988 |
return text
|
|
|
|
| 4174 |
if not page_md:
|
| 4175 |
return page_md
|
| 4176 |
|
| 4177 |
+
# Fresh page: drop any text-layer dedupe caps so the first normalize pass
|
| 4178 |
+
# (before _patch_ocr_with_textlayer) does not see stale keys from a prior page.
|
| 4179 |
_tl_freq_context.clear()
|
| 4180 |
|
| 4181 |
page_md = normalize_money_glyphs(page_md)
|
|
|
|
| 4293 |
|
| 4294 |
all_pages = []
|
| 4295 |
for page_num, page_result in enumerate(results):
|
| 4296 |
+
# Header normalize runs before body patch; do not reuse prior page caps.
|
| 4297 |
_tl_freq_context.clear()
|
| 4298 |
page_md, regions = get_page_md_and_regions(page_result)
|
| 4299 |
img_h = page_heights[page_num] if page_num < len(page_heights) else 1000
|