Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -26,6 +26,8 @@ Primary goals for reconcile rate:
|
|
| 26 |
- Fix 14: UCB 3-col tables where Beginning Balance amount is fused into description and Amount is $0.00
|
| 27 |
- Fix 15: UCB (and similar) 3-col rows where Date+Description are fused in col1 and Amount sits in col2 with col3 empty
|
| 28 |
- Fix 16: remove duplicate rows within a single Date|Description|Amount table; multi-pass table dedupe + final doc-wide dedupe
|
|
|
|
|
|
|
| 29 |
- Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
|
| 30 |
(later subset of earlier, earlier subset of later, or identical row sets)
|
| 31 |
"""
|
|
@@ -3154,6 +3156,116 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
|
|
| 3154 |
return out if len(out) < len(grid) else grid
|
| 3155 |
|
| 3156 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3157 |
def _dedupe_subset_transaction_tables_html(text: str) -> str:
|
| 3158 |
"""
|
| 3159 |
Remove duplicate 3-column Date|Description|Amount tables:
|
|
@@ -3285,6 +3397,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
|
|
| 3285 |
stabilized = "\n\n".join(out_blocks)
|
| 3286 |
stabilized = convert_plaintext_bank_sections(stabilized)
|
| 3287 |
stabilized = normalize_html_tables(stabilized)
|
|
|
|
| 3288 |
stabilized = _dedupe_subset_transaction_tables_html(stabilized)
|
| 3289 |
return close_unclosed_html(stabilized)
|
| 3290 |
|
|
@@ -3412,6 +3525,7 @@ def run_ocr(uploaded_file):
|
|
| 3412 |
# _patch_ocr_with_textlayer may append Case B rows from the PDF text layer
|
| 3413 |
# with fused UCB-style cells; run the same HTML table pass again (Fix 13/14).
|
| 3414 |
stabilized = normalize_html_tables(stabilized)
|
|
|
|
| 3415 |
stabilized = _dedupe_subset_transaction_tables_html(stabilized)
|
| 3416 |
elif is_pdf and is_nfcu_doc:
|
| 3417 |
# Navy-only additive fix: rebuild malformed summary table from
|
|
@@ -3491,8 +3605,9 @@ def run_ocr(uploaded_file):
|
|
| 3491 |
all_pages.append("\n\n".join(parts))
|
| 3492 |
|
| 3493 |
merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
|
| 3494 |
-
# Cross-page:
|
| 3495 |
if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
|
|
|
|
| 3496 |
merged = _dedupe_subset_transaction_tables_html(merged)
|
| 3497 |
return merged
|
| 3498 |
|
|
|
|
| 26 |
- Fix 14: UCB 3-col tables where Beginning Balance amount is fused into description and Amount is $0.00
|
| 27 |
- Fix 15: UCB (and similar) 3-col rows where Date+Description are fused in col1 and Amount sits in col2 with col3 empty
|
| 28 |
- Fix 16: remove duplicate rows within a single Date|Description|Amount table; multi-pass table dedupe + final doc-wide dedupe
|
| 29 |
+
- Fix 17: UCB page 3 — split one merged table under Deposits (continued) into Deposits + Electronic Credits +
|
| 30 |
+
Electronic Debits (PDF text order); drop orphan empty EC/ED headers that followed the merged table
|
| 31 |
- Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
|
| 32 |
(later subset of earlier, earlier subset of later, or identical row sets)
|
| 33 |
"""
|
|
|
|
| 3156 |
return out if len(out) < len(grid) else grid
|
| 3157 |
|
| 3158 |
|
| 3159 |
+
def _row_text_full_ucb(grid, r: int) -> str:
|
| 3160 |
+
"""Lowercased join of all cells in row (for UCB section split heuristics)."""
|
| 3161 |
+
if not grid or r < 0 or r >= len(grid):
|
| 3162 |
+
return ""
|
| 3163 |
+
parts = [str(c or "").strip().lower() for c in grid[r]]
|
| 3164 |
+
return " ".join(parts)
|
| 3165 |
+
|
| 3166 |
+
|
| 3167 |
+
def _is_itm_deposit_row_ucb(grid, r: int) -> bool:
|
| 3168 |
+
if not grid or r >= len(grid) or len(grid[r]) < 2:
|
| 3169 |
+
return False
|
| 3170 |
+
desc = str(grid[r][1] or "").strip().lower()
|
| 3171 |
+
return "itm deposit" in desc
|
| 3172 |
+
|
| 3173 |
+
|
| 3174 |
+
def _ucb_verify_electronic_credits_rows(grid, ec_start: int) -> bool:
|
| 3175 |
+
"""POS Return, Acct Fund (Apple instant), DDA Transfer IN — PDF Electronic Credits block."""
|
| 3176 |
+
if ec_start + 2 >= len(grid):
|
| 3177 |
+
return False
|
| 3178 |
+
t0 = _row_text_full_ucb(grid, ec_start)
|
| 3179 |
+
t1 = _row_text_full_ucb(grid, ec_start + 1)
|
| 3180 |
+
t2 = _row_text_full_ucb(grid, ec_start + 2)
|
| 3181 |
+
ok0 = "pos return" in t0 or ("pos" in t0 and "return" in t0)
|
| 3182 |
+
ok1 = "acct fund" in t1 or ("apple" in t1 and "inst" in t1)
|
| 3183 |
+
ok2 = "dda transfer" in t2 and "in" in t2
|
| 3184 |
+
return bool(ok0 and ok1 and ok2)
|
| 3185 |
+
|
| 3186 |
+
|
| 3187 |
+
def _ucb_split_grid_deposits_ec_ed(grid):
|
| 3188 |
+
"""
|
| 3189 |
+
Return (grid_dep, grid_ec, grid_ed) each with same Date|Description|Amount header,
|
| 3190 |
+
or None if this does not look like the merged UCB page-3 pattern.
|
| 3191 |
+
"""
|
| 3192 |
+
if not grid or len(grid) < 2:
|
| 3193 |
+
return None
|
| 3194 |
+
if not _is_da3_header(grid):
|
| 3195 |
+
return None
|
| 3196 |
+
data_start = 1
|
| 3197 |
+
i = data_start
|
| 3198 |
+
while i < len(grid) and _is_itm_deposit_row_ucb(grid, i):
|
| 3199 |
+
i += 1
|
| 3200 |
+
if i == data_start:
|
| 3201 |
+
return None
|
| 3202 |
+
if i + 3 > len(grid):
|
| 3203 |
+
return None
|
| 3204 |
+
if not _ucb_verify_electronic_credits_rows(grid, i):
|
| 3205 |
+
return None
|
| 3206 |
+
ec_end = i + 3
|
| 3207 |
+
if ec_end >= len(grid):
|
| 3208 |
+
return None
|
| 3209 |
+
hdr = [list(grid[0])]
|
| 3210 |
+
g_dep = hdr + grid[data_start:i]
|
| 3211 |
+
g_ec = hdr + grid[i:ec_end]
|
| 3212 |
+
g_ed = hdr + grid[ec_end:]
|
| 3213 |
+
if len(g_ed) <= 1:
|
| 3214 |
+
return None
|
| 3215 |
+
return g_dep, g_ec, g_ed
|
| 3216 |
+
|
| 3217 |
+
|
| 3218 |
+
def _try_split_ucb_deposits_table_to_three(prefix: str, table_html: str):
|
| 3219 |
+
"""
|
| 3220 |
+
Build HTML: Deposits (continued) + table, Electronic Credits + table, Electronic Debits + table.
|
| 3221 |
+
Returns None if split heuristics do not match.
|
| 3222 |
+
"""
|
| 3223 |
+
try:
|
| 3224 |
+
p = TableGridParser()
|
| 3225 |
+
p.feed(table_html)
|
| 3226 |
+
grid = _build_grid(p.rows)
|
| 3227 |
+
except Exception:
|
| 3228 |
+
return None
|
| 3229 |
+
triple = _ucb_split_grid_deposits_ec_ed(grid)
|
| 3230 |
+
if not triple:
|
| 3231 |
+
return None
|
| 3232 |
+
g_dep, g_ec, g_ed = triple
|
| 3233 |
+
sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
|
| 3234 |
+
parts = [prefix.rstrip(), _grid_to_html(g_dep), sect.format(title="Electronic Credits"), _grid_to_html(g_ec)]
|
| 3235 |
+
parts.append(sect.format(title="Electronic Debits"))
|
| 3236 |
+
parts.append(_grid_to_html(g_ed))
|
| 3237 |
+
return "\n\n".join(parts)
|
| 3238 |
+
|
| 3239 |
+
|
| 3240 |
+
_UCB_DEP_CONT_BLOCK = re.compile(
|
| 3241 |
+
r'(<div\s+align=["\']center["\']\s*>\s*Deposits\s*\(continued\)\s*</div>\s*)'
|
| 3242 |
+
r'(<table[^>]*>.*?</table>)'
|
| 3243 |
+
r'(?:\s*<div\s+align=["\']center["\']\s*>\s*Electronic\s+Credits\s*</div>\s*'
|
| 3244 |
+
r'<div\s+align=["\']center["\']\s*>\s*Electronic\s+Debits\s*</div>)?',
|
| 3245 |
+
re.DOTALL | re.IGNORECASE,
|
| 3246 |
+
)
|
| 3247 |
+
|
| 3248 |
+
|
| 3249 |
+
def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
|
| 3250 |
+
"""
|
| 3251 |
+
Fix 17: Split merged UCB Deposits (continued) table into Deposits + Electronic Credits + Electronic Debits.
|
| 3252 |
+
Removes orphan empty Electronic Credits / Electronic Debits div pairs when replaced by real tables.
|
| 3253 |
+
Safe no-op when heuristics do not match.
|
| 3254 |
+
"""
|
| 3255 |
+
if not text or "deposits (continued)" not in text.lower():
|
| 3256 |
+
return text
|
| 3257 |
+
|
| 3258 |
+
def _repl(m):
|
| 3259 |
+
prefix = m.group(1)
|
| 3260 |
+
table_html = m.group(2)
|
| 3261 |
+
new = _try_split_ucb_deposits_table_to_three(prefix, table_html)
|
| 3262 |
+
if new is None:
|
| 3263 |
+
return m.group(0)
|
| 3264 |
+
return new
|
| 3265 |
+
|
| 3266 |
+
return _UCB_DEP_CONT_BLOCK.sub(_repl, text)
|
| 3267 |
+
|
| 3268 |
+
|
| 3269 |
def _dedupe_subset_transaction_tables_html(text: str) -> str:
|
| 3270 |
"""
|
| 3271 |
Remove duplicate 3-column Date|Description|Amount tables:
|
|
|
|
| 3397 |
stabilized = "\n\n".join(out_blocks)
|
| 3398 |
stabilized = convert_plaintext_bank_sections(stabilized)
|
| 3399 |
stabilized = normalize_html_tables(stabilized)
|
| 3400 |
+
stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
|
| 3401 |
stabilized = _dedupe_subset_transaction_tables_html(stabilized)
|
| 3402 |
return close_unclosed_html(stabilized)
|
| 3403 |
|
|
|
|
| 3525 |
# _patch_ocr_with_textlayer may append Case B rows from the PDF text layer
|
| 3526 |
# with fused UCB-style cells; run the same HTML table pass again (Fix 13/14).
|
| 3527 |
stabilized = normalize_html_tables(stabilized)
|
| 3528 |
+
stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
|
| 3529 |
stabilized = _dedupe_subset_transaction_tables_html(stabilized)
|
| 3530 |
elif is_pdf and is_nfcu_doc:
|
| 3531 |
# Navy-only additive fix: rebuild malformed summary table from
|
|
|
|
| 3605 |
all_pages.append("\n\n".join(parts))
|
| 3606 |
|
| 3607 |
merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
|
| 3608 |
+
# Cross-page: UCB section split + same transaction tables dedupe once on full doc.
|
| 3609 |
if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
|
| 3610 |
+
merged = _split_ucb_deposits_electronic_sections_html(merged)
|
| 3611 |
merged = _dedupe_subset_transaction_tables_html(merged)
|
| 3612 |
return merged
|
| 3613 |
|