rehan953 commited on
Commit
30efa3f
·
verified ·
1 Parent(s): 8a421f8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +116 -1
app.py CHANGED
@@ -26,6 +26,8 @@ Primary goals for reconcile rate:
26
  - Fix 14: UCB 3-col tables where Beginning Balance amount is fused into description and Amount is $0.00
27
  - Fix 15: UCB (and similar) 3-col rows where Date+Description are fused in col1 and Amount sits in col2 with col3 empty
28
  - Fix 16: remove duplicate rows within a single Date|Description|Amount table; multi-pass table dedupe + final doc-wide dedupe
 
 
29
  - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
30
  (later subset of earlier, earlier subset of later, or identical row sets)
31
  """
@@ -3154,6 +3156,116 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
3154
  return out if len(out) < len(grid) else grid
3155
 
3156
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3157
  def _dedupe_subset_transaction_tables_html(text: str) -> str:
3158
  """
3159
  Remove duplicate 3-column Date|Description|Amount tables:
@@ -3285,6 +3397,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
3285
  stabilized = "\n\n".join(out_blocks)
3286
  stabilized = convert_plaintext_bank_sections(stabilized)
3287
  stabilized = normalize_html_tables(stabilized)
 
3288
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
3289
  return close_unclosed_html(stabilized)
3290
 
@@ -3412,6 +3525,7 @@ def run_ocr(uploaded_file):
3412
  # _patch_ocr_with_textlayer may append Case B rows from the PDF text layer
3413
  # with fused UCB-style cells; run the same HTML table pass again (Fix 13/14).
3414
  stabilized = normalize_html_tables(stabilized)
 
3415
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
3416
  elif is_pdf and is_nfcu_doc:
3417
  # Navy-only additive fix: rebuild malformed summary table from
@@ -3491,8 +3605,9 @@ def run_ocr(uploaded_file):
3491
  all_pages.append("\n\n".join(parts))
3492
 
3493
  merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
3494
- # Cross-page: same transaction tables split across page boundaries must dedupe once.
3495
  if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
 
3496
  merged = _dedupe_subset_transaction_tables_html(merged)
3497
  return merged
3498
 
 
26
  - Fix 14: UCB 3-col tables where Beginning Balance amount is fused into description and Amount is $0.00
27
  - Fix 15: UCB (and similar) 3-col rows where Date+Description are fused in col1 and Amount sits in col2 with col3 empty
28
  - Fix 16: remove duplicate rows within a single Date|Description|Amount table; multi-pass table dedupe + final doc-wide dedupe
29
+ - Fix 17: UCB page 3 — split one merged table under Deposits (continued) into Deposits + Electronic Credits +
30
+ Electronic Debits (PDF text order); drop orphan empty EC/ED headers that followed the merged table
31
  - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
32
  (later subset of earlier, earlier subset of later, or identical row sets)
33
  """
 
3156
  return out if len(out) < len(grid) else grid
3157
 
3158
 
3159
+ def _row_text_full_ucb(grid, r: int) -> str:
3160
+ """Lowercased join of all cells in row (for UCB section split heuristics)."""
3161
+ if not grid or r < 0 or r >= len(grid):
3162
+ return ""
3163
+ parts = [str(c or "").strip().lower() for c in grid[r]]
3164
+ return " ".join(parts)
3165
+
3166
+
3167
+ def _is_itm_deposit_row_ucb(grid, r: int) -> bool:
3168
+ if not grid or r >= len(grid) or len(grid[r]) < 2:
3169
+ return False
3170
+ desc = str(grid[r][1] or "").strip().lower()
3171
+ return "itm deposit" in desc
3172
+
3173
+
3174
+ def _ucb_verify_electronic_credits_rows(grid, ec_start: int) -> bool:
3175
+ """POS Return, Acct Fund (Apple instant), DDA Transfer IN — PDF Electronic Credits block."""
3176
+ if ec_start + 2 >= len(grid):
3177
+ return False
3178
+ t0 = _row_text_full_ucb(grid, ec_start)
3179
+ t1 = _row_text_full_ucb(grid, ec_start + 1)
3180
+ t2 = _row_text_full_ucb(grid, ec_start + 2)
3181
+ ok0 = "pos return" in t0 or ("pos" in t0 and "return" in t0)
3182
+ ok1 = "acct fund" in t1 or ("apple" in t1 and "inst" in t1)
3183
+ ok2 = "dda transfer" in t2 and "in" in t2
3184
+ return bool(ok0 and ok1 and ok2)
3185
+
3186
+
3187
+ def _ucb_split_grid_deposits_ec_ed(grid):
3188
+ """
3189
+ Return (grid_dep, grid_ec, grid_ed) each with same Date|Description|Amount header,
3190
+ or None if this does not look like the merged UCB page-3 pattern.
3191
+ """
3192
+ if not grid or len(grid) < 2:
3193
+ return None
3194
+ if not _is_da3_header(grid):
3195
+ return None
3196
+ data_start = 1
3197
+ i = data_start
3198
+ while i < len(grid) and _is_itm_deposit_row_ucb(grid, i):
3199
+ i += 1
3200
+ if i == data_start:
3201
+ return None
3202
+ if i + 3 > len(grid):
3203
+ return None
3204
+ if not _ucb_verify_electronic_credits_rows(grid, i):
3205
+ return None
3206
+ ec_end = i + 3
3207
+ if ec_end >= len(grid):
3208
+ return None
3209
+ hdr = [list(grid[0])]
3210
+ g_dep = hdr + grid[data_start:i]
3211
+ g_ec = hdr + grid[i:ec_end]
3212
+ g_ed = hdr + grid[ec_end:]
3213
+ if len(g_ed) <= 1:
3214
+ return None
3215
+ return g_dep, g_ec, g_ed
3216
+
3217
+
3218
+ def _try_split_ucb_deposits_table_to_three(prefix: str, table_html: str):
3219
+ """
3220
+ Build HTML: Deposits (continued) + table, Electronic Credits + table, Electronic Debits + table.
3221
+ Returns None if split heuristics do not match.
3222
+ """
3223
+ try:
3224
+ p = TableGridParser()
3225
+ p.feed(table_html)
3226
+ grid = _build_grid(p.rows)
3227
+ except Exception:
3228
+ return None
3229
+ triple = _ucb_split_grid_deposits_ec_ed(grid)
3230
+ if not triple:
3231
+ return None
3232
+ g_dep, g_ec, g_ed = triple
3233
+ sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
3234
+ parts = [prefix.rstrip(), _grid_to_html(g_dep), sect.format(title="Electronic Credits"), _grid_to_html(g_ec)]
3235
+ parts.append(sect.format(title="Electronic Debits"))
3236
+ parts.append(_grid_to_html(g_ed))
3237
+ return "\n\n".join(parts)
3238
+
3239
+
3240
+ _UCB_DEP_CONT_BLOCK = re.compile(
3241
+ r'(<div\s+align=["\']center["\']\s*>\s*Deposits\s*\(continued\)\s*</div>\s*)'
3242
+ r'(<table[^>]*>.*?</table>)'
3243
+ r'(?:\s*<div\s+align=["\']center["\']\s*>\s*Electronic\s+Credits\s*</div>\s*'
3244
+ r'<div\s+align=["\']center["\']\s*>\s*Electronic\s+Debits\s*</div>)?',
3245
+ re.DOTALL | re.IGNORECASE,
3246
+ )
3247
+
3248
+
3249
+ def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
3250
+ """
3251
+ Fix 17: Split merged UCB Deposits (continued) table into Deposits + Electronic Credits + Electronic Debits.
3252
+ Removes orphan empty Electronic Credits / Electronic Debits div pairs when replaced by real tables.
3253
+ Safe no-op when heuristics do not match.
3254
+ """
3255
+ if not text or "deposits (continued)" not in text.lower():
3256
+ return text
3257
+
3258
+ def _repl(m):
3259
+ prefix = m.group(1)
3260
+ table_html = m.group(2)
3261
+ new = _try_split_ucb_deposits_table_to_three(prefix, table_html)
3262
+ if new is None:
3263
+ return m.group(0)
3264
+ return new
3265
+
3266
+ return _UCB_DEP_CONT_BLOCK.sub(_repl, text)
3267
+
3268
+
3269
  def _dedupe_subset_transaction_tables_html(text: str) -> str:
3270
  """
3271
  Remove duplicate 3-column Date|Description|Amount tables:
 
3397
  stabilized = "\n\n".join(out_blocks)
3398
  stabilized = convert_plaintext_bank_sections(stabilized)
3399
  stabilized = normalize_html_tables(stabilized)
3400
+ stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
3401
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
3402
  return close_unclosed_html(stabilized)
3403
 
 
3525
  # _patch_ocr_with_textlayer may append Case B rows from the PDF text layer
3526
  # with fused UCB-style cells; run the same HTML table pass again (Fix 13/14).
3527
  stabilized = normalize_html_tables(stabilized)
3528
+ stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
3529
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
3530
  elif is_pdf and is_nfcu_doc:
3531
  # Navy-only additive fix: rebuild malformed summary table from
 
3605
  all_pages.append("\n\n".join(parts))
3606
 
3607
  merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
3608
+ # Cross-page: UCB section split + same transaction tables dedupe once on full doc.
3609
  if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
3610
+ merged = _split_ucb_deposits_electronic_sections_html(merged)
3611
  merged = _dedupe_subset_transaction_tables_html(merged)
3612
  return merged
3613