rehan953 commited on
Commit
2a03bb8
·
verified ·
1 Parent(s): de55a85

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +232 -105
app.py CHANGED
@@ -36,16 +36,18 @@ Primary goals for reconcile rate:
36
  debit pattern) and at least one ITM DEPOSIT row exists, OCR stacked headings wrong — emit
37
  Electronic Credits, then the table, then Electronic Debits (section continues below). Safe no-op
38
  if the Debits heading was not before the table or any row looks like a card/ATM debit.
39
- GUARD: Do NOT fire Fix 17c if ALL rows are ITM DEPOSIT — that indicates a Deposits (continued)
40
- block that was incorrectly preceded by EC/ED headings, not a genuine Electronic Credits table.
41
  - Fix 17c-html: Same as 17c using raw HTML cues when TableGridParser/header normalization would skip the table.
42
- Same ITM-only guard applied.
43
  - Fix 17d: Drop a second Electronic Debits block + DA3 table when its rows are a loose (date+amount)
44
  multiset subset of the immediately preceding DA3 table (OCR duplicate tail / wrong split).
45
- GUARD: Do NOT drop a small table if it contains genuine credit-side rows (POS Return,
46
- Acct Fund INST, DDA Transfer IN)these are misplaced EC rows, not ED duplicates.
 
 
 
 
 
47
  - Fix 17e: Repeat EC/ED fused + 17d passes until stable (handles chained replacements). Centered section titles
48
- match both align="center" and style="text-align:center" on divs.
49
  - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
50
  (later subset of earlier, earlier subset of later, or identical row sets)
51
  - Fix 18: adjacent DA3 tables — if the last K data rows of table N match the first K rows of table N+1
@@ -92,9 +94,6 @@ Primary goals for reconcile rate:
92
  earlier larger DA3 table — catches OCR-truncated description duplicates such as
93
  the BoA service-fee page where a VIDDYOZE -1.11 fragment table appears after the
94
  full service-fee table with a longer description string.
95
- - Fix 17c/17c-html/17d GUARD (UCB page 3): Prevent misclassification of "Deposits (continued)"
96
- ITM DEPOSIT blocks as Electronic Credits, and prevent Fix 17d from dropping the
97
- real Electronic Credits table (POS Return / Acct Fund INST / DDA Transfer IN).
98
  """
99
 
100
  # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -1123,6 +1122,10 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1123
  key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
1124
  tl_freq[key] = tl_freq.get(key, 0) + 1
1125
 
 
 
 
 
1126
  global _tl_freq_context
1127
  tl_dedupe = {}
1128
  for r in tl_rows:
@@ -2682,7 +2685,7 @@ def _transaction_row_dedupe_key(cells):
2682
  date_s = re.sub(r"\s+", "", date_s)
2683
  amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
2684
  desc_s = html.unescape(desc_s)
2685
- desc_s = desc_s.replace("`", "'").replace("\u2019", "'")
2686
  desc_s = re.sub(r"\s+", "", desc_s.lower())
2687
  return (date_s, desc_s, amt_s)
2688
 
@@ -2751,6 +2754,11 @@ _DA3_MONEY_TOKEN = re.compile(
2751
  )
2752
  _MSFT_PIN_G_REF = re.compile(r"MICROSOFT#(G\d{6,16})", re.IGNORECASE)
2753
 
 
 
 
 
 
2754
  _tl_freq_context: dict = {}
2755
 
2756
  _DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
@@ -2841,6 +2849,12 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
2841
  break
2842
  j += 1
2843
  run_len = j - i
 
 
 
 
 
 
2844
  tl_confirmed = _tl_freq_context.get(k, 0)
2845
  cap = max(tl_confirmed, 2) if tl_confirmed > 0 else 2
2846
  take = min(run_len, cap)
@@ -2935,38 +2949,10 @@ def _ucb_has_itm_deposit_data_row(grid) -> bool:
2935
  return False
2936
 
2937
 
2938
- def _ucb_all_rows_are_itm_deposit(grid) -> bool:
2939
- """
2940
- Returns True if ALL data rows (row index >= 1) in a DA3 grid are
2941
- ITM DEPOSIT rows. Used to guard Fix 17c / 17c-html: a table that is
2942
- exclusively ITM DEPOSIT entries is a "Deposits (continued)" block,
2943
- NOT an Electronic Credits table, even when it has no strong-debit rows.
2944
- """
2945
- if not grid or len(grid) < 2:
2946
- return False
2947
- data_rows = [grid[r] for r in range(1, len(grid))
2948
- if any(str(c or "").strip() for c in grid[r])]
2949
- if not data_rows:
2950
- return False
2951
- return all(_is_itm_deposit_row_ucb(grid, r) for r in range(1, len(grid))
2952
- if any(str(c or "").strip() for c in grid[r]))
2953
-
2954
-
2955
  def _ucb_table_credits_only_no_strong_debits(grid) -> bool:
2956
- """
2957
- True if DA3 table has real data rows but none match strong
2958
- electronic-debit patterns AND at least one ITM DEPOSIT row exists.
2959
-
2960
- FIX 17c GUARD: Returns False when ALL data rows are ITM DEPOSIT.
2961
- That case is a "Deposits (continued)" block — not Electronic Credits —
2962
- so Fix 17c must NOT reorder it under an EC heading.
2963
- """
2964
  if not grid or len(grid) < 2 or not _is_da3_header(grid):
2965
  return False
2966
- # Guard: if every row is ITM DEPOSIT, this is Deposits (continued),
2967
- # not Electronic Credits. Do not trigger Fix 17c.
2968
- if _ucb_all_rows_are_itm_deposit(grid):
2969
- return False
2970
  has_data = False
2971
  for r in range(1, len(grid)):
2972
  row_txt = _row_text_full_ucb(grid, r).strip()
@@ -2982,9 +2968,6 @@ def _ucb_html_table_body_looks_itm_credits_only(table_html: str) -> bool:
2982
  """
2983
  Detect ITM-only credit block without relying on parsed grid headers (Fix 17c-html).
2984
  Conservative: require ITM deposit text and absence of common electronic-debit verbs in the HTML.
2985
-
2986
- FIX 17c-html GUARD: Returns False when the table contains ONLY ITM DEPOSIT rows
2987
- (no other description text), as that indicates a Deposits (continued) block.
2988
  """
2989
  if not table_html or "<table" not in table_html.lower():
2990
  return False
@@ -2995,30 +2978,6 @@ def _ucb_html_table_body_looks_itm_credits_only(table_html: str) -> bool:
2995
  return False
2996
  if "recur payment" in low or "debit card" in low:
2997
  return False
2998
- # Additional guard: if every <td> description cell only contains "ITM DEPOSIT"
2999
- # (no other transaction types), treat this as Deposits (continued), not EC.
3000
- # Extract description cell texts from <td> elements (skip header row <th>).
3001
- td_texts = re.findall(r"<td>([^<]*)</td>", table_html, re.IGNORECASE)
3002
- if td_texts:
3003
- # td_texts from a DA3 table come in triplets: date, desc, amount
3004
- # Check if any non-date, non-amount cell has text other than "ITM DEPOSIT"
3005
- non_itm_desc = False
3006
- for i, txt in enumerate(td_texts):
3007
- txt_stripped = txt.strip().lower()
3008
- if not txt_stripped:
3009
- continue
3010
- # Skip date-like tokens and amount-like tokens
3011
- if re.match(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$", txt_stripped):
3012
- continue
3013
- if re.match(r"^\$?[\d,]+\.\d{2}$", txt_stripped):
3014
- continue
3015
- # It's a description cell
3016
- if "itm deposit" not in txt_stripped and "itm dep" not in txt_stripped:
3017
- non_itm_desc = True
3018
- break
3019
- if not non_itm_desc:
3020
- # All description cells are ITM DEPOSIT — this is Deposits (continued)
3021
- return False
3022
  return True
3023
 
3024
 
@@ -3076,7 +3035,6 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
3076
 
3077
  # Fix 17c: headings stacked as EC / ED / table but rows are all credits (e.g. ITM-only) —
3078
  # put the table under Electronic Credits, then leave Electronic Debits for continuation.
3079
- # GUARD: Do NOT fire if all rows are ITM DEPOSIT (that is Deposits continued, not EC).
3080
  if had_ed_heading_immediately_before_table and _ucb_table_credits_only_no_strong_debits(grid):
3081
  sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
3082
  parts = []
@@ -3092,7 +3050,6 @@ def _try_split_ucb_ec_ed_from_fused_table_html(
3092
  return "\n\n".join(parts)
3093
 
3094
  # Fix 17c-html: same reorder when grid header checks fail but raw HTML is clearly ITM-only credits.
3095
- # GUARD: _ucb_html_table_body_looks_itm_credits_only now returns False for pure ITM DEPOSIT tables.
3096
  if had_ed_heading_immediately_before_table and _ucb_html_table_body_looks_itm_credits_only(table_html):
3097
  sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
3098
  parts = []
@@ -3119,6 +3076,8 @@ _UCB_CENTER_DIV = (
3119
  r')'
3120
  )
3121
 
 
 
3122
  _UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
3123
  r'(?P<lead>(?:' + _UCB_CENTER_DIV + r'\s*</div>\s*)?)'
3124
  r'(?P<ecdiv>' + _UCB_CENTER_DIV + r'\s*Electronic\s+Credits\s*</div>\s*)'
@@ -3128,32 +3087,12 @@ _UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
3128
  )
3129
 
3130
 
3131
- def _ucb_table_html_has_genuine_credit_rows(table_html: str) -> bool:
3132
- """
3133
- Returns True if the table contains rows that are clearly credit-side
3134
- (POS Return, Acct Fund INST/CUPERTINO, DDA Transfer IN).
3135
- Used by Fix 17d guard to prevent dropping the real EC table.
3136
- """
3137
- if not table_html:
3138
- return False
3139
- low = table_html.lower()
3140
- if "pos return" in low:
3141
- return True
3142
- if "acct fund" in low and ("inst" in low or "cupertino" in low):
3143
- return True
3144
- if "dda transfer" in low and "in" in low:
3145
- return True
3146
- return False
3147
-
3148
-
3149
  def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
3150
  """
3151
- Fix 17d: Remove a duplicate <div>Electronic Debits</div> + <table> when that table's rows are a
3152
  loose (date, amount) multiset subset of the DA3 table that appears immediately before this block.
3153
-
3154
- GUARD: Do NOT drop the table if it contains genuine credit-side rows (POS Return,
3155
- Acct Fund INST/CUPERTINO, DDA Transfer IN). Those rows belong under Electronic Credits
3156
- and must not be silently discarded as ED duplicates.
3157
  """
3158
  if not text or "electronic debits" not in text.lower():
3159
  return text
@@ -3165,6 +3104,7 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
3165
  )
3166
  tbl_pat = re.compile(r'<table[^>]*>.*?</table>', re.DOTALL | re.IGNORECASE)
3167
  _DATE_NONEMPTY = re.compile(r"^\d{1,2}[/\-]\d{2}")
 
3168
  _AMT_STRIPPED_OK = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
3169
 
3170
  def _loose_da_key_cells(cells):
@@ -3193,6 +3133,35 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
3193
  keys.append(k)
3194
  return keys
3195
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3196
  for _ in range(12):
3197
  changed = False
3198
  for m in ed_block.finditer(text):
@@ -3206,29 +3175,156 @@ def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
3206
  g_frag = _parse_da3_grid_from_table_html(m.group("tbl"))
3207
  kb = _loose_keys_from_grid(g_big)
3208
  kf = _loose_keys_from_grid(g_frag)
3209
- if not kb or not kf:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3210
  continue
3211
- if len(kf) > 24 or len(kf) >= len(kb):
 
 
 
 
 
 
 
 
 
 
 
3212
  continue
3213
- cb, cf = Counter(kb), Counter(kf)
3214
- if not all(cf[k] <= cb.get(k, 0) for k in cf):
 
 
 
 
 
 
3215
  continue
3216
- # GUARD: Do not drop a table that contains genuine credit-side rows.
3217
- # These rows belong under Electronic Credits, not as ED duplicates.
3218
- if _ucb_table_html_has_genuine_credit_rows(m.group("tbl")):
3219
- log.info(
3220
- "Fix 17d: skipping drop of table with genuine credit rows "
3221
- "(POS Return / Acct Fund INST / DDA Transfer IN)"
3222
- )
3223
  continue
3224
- text = text[: m.start()] + text[m.end() :]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3225
  changed = True
3226
  break
3227
  if not changed:
3228
  break
3229
  return text
3230
  except Exception as e:
3231
- log.warning("_ucb_strip_redundant_ed_subset_fragment failed: %s", e)
3232
  return text
3233
 
3234
 
@@ -3290,10 +3386,11 @@ def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
3290
  return new
3291
 
3292
  text = _UCB_DEP_CONT_BLOCK.sub(_repl, text)
3293
- # Fix 17e: repeat fused EC/ED fix + duplicate ED strip until stable (one pass can enable the next).
3294
  for _ in range(16):
3295
  nxt = _split_ucb_fused_ec_ed_headings_single_table_html(text)
3296
  nxt = _ucb_strip_redundant_ed_subset_fragment(nxt)
 
3297
  if nxt == text:
3298
  break
3299
  text = nxt
@@ -3505,6 +3602,21 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
3505
  """
3506
  Fix 30: Drop small DA3 table fragments (1-5 data rows) that are a loose
3507
  subset of an earlier larger DA3 table, matched by (date, amount) key alone.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3508
  """
3509
  if not text or "<table" not in text.lower():
3510
  return text
@@ -3518,6 +3630,7 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
3518
  _AMT_NONEMPTY = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
3519
 
3520
  def _da_key(cells):
 
3521
  parts = [str(c or "").strip() for c in cells[:3]]
3522
  while len(parts) < 3:
3523
  parts.append("")
@@ -3526,6 +3639,7 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
3526
  amt_s = amt_s.replace("\u2212", "-").replace("\u2013", "-")
3527
  return (date_s, amt_s)
3528
 
 
3529
  da3_info = []
3530
  for m in matches:
3531
  g = _parse_da3_grid_from_table_html(m.group(0))
@@ -3559,6 +3673,8 @@ def _dedupe_da3_by_date_amount_loose(text: str) -> str:
3559
  if tj["n"] <= ti["n"]:
3560
  continue
3561
  big_keys = Counter(tj["keys"])
 
 
3562
  if all(frag_keys[k] <= big_keys.get(k, 0) for k in frag_keys):
3563
  drop_set.add(ti["match"].start())
3564
  log.info(
@@ -3859,6 +3975,14 @@ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
3859
  """
3860
  Fix 28: When OCR prepends the rows of a preceding credits/deposits DA3 table
3861
  onto the top of a debits/withdrawals DA3 table, strip that prefix.
 
 
 
 
 
 
 
 
3862
  """
3863
  if not text or "<table" not in text.lower():
3864
  return text
@@ -4050,6 +4174,8 @@ def stabilize_tables_and_text(page_md: str) -> str:
4050
  if not page_md:
4051
  return page_md
4052
 
 
 
4053
  _tl_freq_context.clear()
4054
 
4055
  page_md = normalize_money_glyphs(page_md)
@@ -4167,6 +4293,7 @@ def run_ocr(uploaded_file):
4167
 
4168
  all_pages = []
4169
  for page_num, page_result in enumerate(results):
 
4170
  _tl_freq_context.clear()
4171
  page_md, regions = get_page_md_and_regions(page_result)
4172
  img_h = page_heights[page_num] if page_num < len(page_heights) else 1000
 
36
  debit pattern) and at least one ITM DEPOSIT row exists, OCR stacked headings wrong — emit
37
  Electronic Credits, then the table, then Electronic Debits (section continues below). Safe no-op
38
  if the Debits heading was not before the table or any row looks like a card/ATM debit.
 
 
39
  - Fix 17c-html: Same as 17c using raw HTML cues when TableGridParser/header normalization would skip the table.
 
40
  - Fix 17d: Drop a second Electronic Debits block + DA3 table when its rows are a loose (date+amount)
41
  multiset subset of the immediately preceding DA3 table (OCR duplicate tail / wrong split).
42
+ - Fix 17f: Same as 17d when the duplicate table *starts with* ITM DEPOSIT rows (copied from Electronic Credits)
43
+ then repeats debits from the table above 17d alone misses those because ITM keys are not in the
44
+ debits-only predecessor table. Strip ITM rows from the multiset check, then apply the 17d subset rule.
45
+ - Fix 17g: When 17f cannot drop the whole block (e.g. one orphan debit row only appears in the duplicate tail),
46
+ drop leading ITM rows and each fragment row whose (date, amount) still has count in the previous
47
+ debits table; remove the ED wrapper if the table becomes empty. If a small orphan tail remains
48
+ (≤6 rows), append those rows to the preceding debits table and drop the extra ED heading.
49
  - Fix 17e: Repeat EC/ED fused + 17d passes until stable (handles chained replacements). Centered section titles
50
+ match both align=\"center\" and style=\"text-align:center\" on divs.
51
  - Post-pass: dedupe 3-col Date|Description|Amount tables when one is a duplicate fragment
52
  (later subset of earlier, earlier subset of later, or identical row sets)
53
  - Fix 18: adjacent DA3 tables — if the last K data rows of table N match the first K rows of table N+1
 
94
  earlier larger DA3 table — catches OCR-truncated description duplicates such as
95
  the BoA service-fee page where a VIDDYOZE -1.11 fragment table appears after the
96
  full service-fee table with a longer description string.
 
 
 
97
  """
98
 
99
  # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
 
1122
  key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
1123
  tl_freq[key] = tl_freq.get(key, 0) + 1
1124
 
1125
+ # Share with _dedupe_duplicate_rows_in_da3_table (Fix-20 override).
1126
+ # Keys MUST match _transaction_row_dedupe_key (date/desc/amount, no spaces in
1127
+ # date/desc; amount stripped of $/,/space). The injection map tl_freq uses
1128
+ # _norm() with spaces — different tuple shape — so we keep a parallel count.
1129
  global _tl_freq_context
1130
  tl_dedupe = {}
1131
  for r in tl_rows:
 
2685
  date_s = re.sub(r"\s+", "", date_s)
2686
  amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
2687
  desc_s = html.unescape(desc_s)
2688
+ desc_s = desc_s.replace("`", "'").replace("'", "'")
2689
  desc_s = re.sub(r"\s+", "", desc_s.lower())
2690
  return (date_s, desc_s, amt_s)
2691
 
 
2754
  )
2755
  _MSFT_PIN_G_REF = re.compile(r"MICROSOFT#(G\d{6,16})", re.IGNORECASE)
2756
 
2757
+ # Module-level text-layer frequency context (Fix-20 override).
2758
+ # Set by _patch_ocr_with_textlayer; consumed by normalize_html_tables in run_ocr's
2759
+ # post-patch pass. Cleared at the start of stabilize_tables_and_text (new page)
2760
+ # and on patch failure so keys never bleed across pages.
2761
+ # Keys match _transaction_row_dedupe_key (not the _norm()-based tl_freq map).
2762
  _tl_freq_context: dict = {}
2763
 
2764
  _DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
 
2849
  break
2850
  j += 1
2851
  run_len = j - i
2852
+ # Cap at 2 by default (Fix 20: collapse OCR stutters) but honour
2853
+ # the text-layer-confirmed count when available, so legitimate
2854
+ # repeated bank transactions (e.g. 6x Temporary Credit Reversal
2855
+ # on the same date/amount) are not incorrectly collapsed to 2.
2856
+ # Keys absent from _tl_freq_context still cap at 2 — safe no-op
2857
+ # for scanned PDFs or pages where no text layer was extracted.
2858
  tl_confirmed = _tl_freq_context.get(k, 0)
2859
  cap = max(tl_confirmed, 2) if tl_confirmed > 0 else 2
2860
  take = min(run_len, cap)
 
2949
  return False
2950
 
2951
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2952
  def _ucb_table_credits_only_no_strong_debits(grid) -> bool:
2953
+ """True if DA3 table has real data rows but none match strong electronic-debit patterns."""
 
 
 
 
 
 
 
2954
  if not grid or len(grid) < 2 or not _is_da3_header(grid):
2955
  return False
 
 
 
 
2956
  has_data = False
2957
  for r in range(1, len(grid)):
2958
  row_txt = _row_text_full_ucb(grid, r).strip()
 
2968
  """
2969
  Detect ITM-only credit block without relying on parsed grid headers (Fix 17c-html).
2970
  Conservative: require ITM deposit text and absence of common electronic-debit verbs in the HTML.
 
 
 
2971
  """
2972
  if not table_html or "<table" not in table_html.lower():
2973
  return False
 
2978
  return False
2979
  if "recur payment" in low or "debit card" in low:
2980
  return False
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2981
  return True
2982
 
2983
 
 
3035
 
3036
  # Fix 17c: headings stacked as EC / ED / table but rows are all credits (e.g. ITM-only) —
3037
  # put the table under Electronic Credits, then leave Electronic Debits for continuation.
 
3038
  if had_ed_heading_immediately_before_table and _ucb_table_credits_only_no_strong_debits(grid):
3039
  sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
3040
  parts = []
 
3050
  return "\n\n".join(parts)
3051
 
3052
  # Fix 17c-html: same reorder when grid header checks fail but raw HTML is clearly ITM-only credits.
 
3053
  if had_ed_heading_immediately_before_table and _ucb_html_table_body_looks_itm_credits_only(table_html):
3054
  sect = '<div align="center">\n\n{title}\n\n</div>\n\n'
3055
  parts = []
 
3076
  r')'
3077
  )
3078
 
3079
+ # Optional empty centered banner only; EC/ED title divs are re-emitted after a successful split
3080
+ # so we do not duplicate headings.
3081
  _UCB_EC_ED_SINGLE_FUSED_TABLE = re.compile(
3082
  r'(?P<lead>(?:' + _UCB_CENTER_DIV + r'\s*</div>\s*)?)'
3083
  r'(?P<ecdiv>' + _UCB_CENTER_DIV + r'\s*Electronic\s+Credits\s*</div>\s*)'
 
3087
  )
3088
 
3089
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3090
  def _ucb_strip_redundant_ed_subset_fragment(text: str) -> str:
3091
  """
3092
+ Fix 17d / 17f: Remove a duplicate <div>Electronic Debits</div> + <table> when that table's rows are a
3093
  loose (date, amount) multiset subset of the DA3 table that appears immediately before this block.
3094
+ Fix 17f: If the fragment also contains ITM DEPOSIT rows (mis-copied under Debits), ignore those rows
3095
+ for the subset check so the tail still compares against the preceding debits table.
 
 
3096
  """
3097
  if not text or "electronic debits" not in text.lower():
3098
  return text
 
3104
  )
3105
  tbl_pat = re.compile(r'<table[^>]*>.*?</table>', re.DOTALL | re.IGNORECASE)
3106
  _DATE_NONEMPTY = re.compile(r"^\d{1,2}[/\-]\d{2}")
3107
+ # Stripped amounts may be "2194.19" (no comma) — must not use comma-group regex.
3108
  _AMT_STRIPPED_OK = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
3109
 
3110
  def _loose_da_key_cells(cells):
 
3133
  keys.append(k)
3134
  return keys
3135
 
3136
+ def _loose_keys_from_grid_skip_itm(g):
3137
+ if not g or not _is_da3_header(g):
3138
+ return None
3139
+ keys = []
3140
+ for r in range(1, len(g)):
3141
+ if _is_itm_deposit_row_ucb(g, r):
3142
+ continue
3143
+ cells = [str(c or "").strip() for c in g[r]]
3144
+ if not any(cells):
3145
+ continue
3146
+ k = _loose_da_key_cells(cells)
3147
+ if (
3148
+ _DATE_NONEMPTY.match(k[0])
3149
+ and _AMT_STRIPPED_OK.match(k[1])
3150
+ and "." in k[1]
3151
+ ):
3152
+ keys.append(k)
3153
+ return keys
3154
+
3155
+ def _frag_has_itm_data_row(g):
3156
+ if not g:
3157
+ return False
3158
+ for r in range(1, len(g)):
3159
+ if not any(str(c or "").strip() for c in g[r]):
3160
+ continue
3161
+ if _is_itm_deposit_row_ucb(g, r):
3162
+ return True
3163
+ return False
3164
+
3165
  for _ in range(12):
3166
  changed = False
3167
  for m in ed_block.finditer(text):
 
3175
  g_frag = _parse_da3_grid_from_table_html(m.group("tbl"))
3176
  kb = _loose_keys_from_grid(g_big)
3177
  kf = _loose_keys_from_grid(g_frag)
3178
+ strip = False
3179
+ if kb and kf and len(kf) <= 24 and len(kf) < len(kb):
3180
+ cb, cf = Counter(kb), Counter(kf)
3181
+ if all(cf[k] <= cb.get(k, 0) for k in cf):
3182
+ strip = True
3183
+ # Fix 17f: ITM rows duplicated from Credits; debit tail ⊆ previous debits table
3184
+ if (
3185
+ not strip
3186
+ and kb
3187
+ and g_frag
3188
+ and _frag_has_itm_data_row(g_frag)
3189
+ ):
3190
+ kf2 = _loose_keys_from_grid_skip_itm(g_frag)
3191
+ if (
3192
+ kf2
3193
+ and len(kf2) <= 24
3194
+ and len(kf2) < len(kb)
3195
+ ):
3196
+ cb, cf2 = Counter(kb), Counter(kf2)
3197
+ if all(cf2[k] <= cb.get(k, 0) for k in cf2):
3198
+ strip = True
3199
+ if strip:
3200
+ text = text[: m.start()] + text[m.end() :]
3201
+ changed = True
3202
+ break
3203
+ if not changed:
3204
+ break
3205
+ return text
3206
+ except Exception as e:
3207
+ log.warning("_ucb_strip_redundant_ed_subset_fragment failed: %s", e)
3208
+ return text
3209
+
3210
+
3211
+ def _ucb_prune_ed_fragment_dupes_vs_prev_table(text: str) -> str:
3212
+ """
3213
+ Fix 17g: Under a second Electronic Debits + DA3 table, drop leading ITM rows and rows that duplicate
3214
+ the immediately preceding debits table by (date, amount). Keeps orphan rows (same key absent from prev).
3215
+ Removes the whole block if only the header remains. If ≤6 orphan rows remain, merge them into the
3216
+ preceding table and remove this ED block.
3217
+ """
3218
+ if not text or "electronic debits" not in text.lower():
3219
+ return text
3220
+ try:
3221
+ ed_block = re.compile(
3222
+ _UCB_CENTER_DIV + r"\s*Electronic\s+Debits\s*</div>\s*"
3223
+ r"(?P<tbl><table[^>]*>.*?</table>)",
3224
+ re.DOTALL | re.IGNORECASE,
3225
+ )
3226
+ tbl_pat = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
3227
+ _dn = re.compile(r"^\d{1,2}[/\-]\d{2}")
3228
+ _amt_ok = re.compile(r"^-?\d+(?:\.\d{1,4})?-?$")
3229
+
3230
+ def _key_cells(cells):
3231
+ parts = [str(c or "").strip() for c in cells[:3]]
3232
+ while len(parts) < 3:
3233
+ parts.append("")
3234
+ date_s = re.sub(r"\s+", "", parts[0])
3235
+ amt_s = re.sub(r"[\s$,]", "", parts[2]).lower()
3236
+ amt_s = amt_s.replace("\u2212", "-").replace("\u2013", "-")
3237
+ return date_s, amt_s
3238
+
3239
+ def _row_loose_key(grid, r):
3240
+ cells = [str(c or "").strip() for c in grid[r]]
3241
+ if not any(cells):
3242
+ return None
3243
+ k = _key_cells(cells)
3244
+ if not (_dn.match(k[0]) and _amt_ok.match(k[1]) and "." in k[1]):
3245
+ return None
3246
+ return k
3247
+
3248
+ def _has_itm_data(grid):
3249
+ for r in range(1, len(grid)):
3250
+ if not any(str(c or "").strip() for c in grid[r]):
3251
  continue
3252
+ if _is_itm_deposit_row_ucb(grid, r):
3253
+ return True
3254
+ return False
3255
+
3256
+ for _ in range(12):
3257
+ changed = False
3258
+ for m in ed_block.finditer(text):
3259
+ before = text[: m.start()]
3260
+ prev_tbl_m = None
3261
+ for tm in tbl_pat.finditer(before):
3262
+ prev_tbl_m = tm
3263
+ if prev_tbl_m is None:
3264
  continue
3265
+ g_big = _parse_da3_grid_from_table_html(prev_tbl_m.group(0))
3266
+ g_frag = _parse_da3_grid_from_table_html(m.group("tbl"))
3267
+ if (
3268
+ not g_big
3269
+ or not g_frag
3270
+ or not _is_da3_header(g_big)
3271
+ or not _is_da3_header(g_frag)
3272
+ ):
3273
  continue
3274
+ if not _has_itm_data(g_frag):
 
 
 
 
 
 
3275
  continue
3276
+ cb = Counter()
3277
+ for r in range(1, len(g_big)):
3278
+ kk = _row_loose_key(g_big, r)
3279
+ if kk:
3280
+ cb[kk] += 1
3281
+ new_rows = []
3282
+ seen_non_itm = False
3283
+ for r in range(1, len(g_frag)):
3284
+ if not any(str(c or "").strip() for c in g_frag[r]):
3285
+ continue
3286
+ if _is_itm_deposit_row_ucb(g_frag, r):
3287
+ if not seen_non_itm:
3288
+ continue
3289
+ continue
3290
+ seen_non_itm = True
3291
+ kk = _row_loose_key(g_frag, r)
3292
+ if kk and cb.get(kk, 0) > 0:
3293
+ cb[kk] -= 1
3294
+ continue
3295
+ new_rows.append(list(g_frag[r]))
3296
+ if not new_rows:
3297
+ text = text[: m.start()] + text[m.end() :]
3298
+ changed = True
3299
+ break
3300
+ if len(new_rows) <= 6:
3301
+ merged = [list(g_big[0])] + [list(r) for r in g_big[1:]] + new_rows
3302
+ max_c = max(len(r) for r in merged)
3303
+ for row in merged:
3304
+ while len(row) < max_c:
3305
+ row.append("")
3306
+ merged_html = _grid_to_html(merged)
3307
+ ps, pe = prev_tbl_m.start(), prev_tbl_m.end()
3308
+ text = text[:ps] + merged_html + text[pe : m.start()] + text[m.end() :]
3309
+ changed = True
3310
+ break
3311
+ hdr = [list(g_frag[0])]
3312
+ max_c = max(len(hdr[0]), max((len(x) for x in new_rows), default=0))
3313
+ for row in hdr + new_rows:
3314
+ while len(row) < max_c:
3315
+ row.append("")
3316
+ new_tbl = _grid_to_html(hdr + new_rows)
3317
+ full = m.group(0)
3318
+ tbl = m.group("tbl")
3319
+ prefix = full[: full.index(tbl)]
3320
+ text = text[: m.start()] + prefix + new_tbl + text[m.end() :]
3321
  changed = True
3322
  break
3323
  if not changed:
3324
  break
3325
  return text
3326
  except Exception as e:
3327
+ log.warning("_ucb_prune_ed_fragment_dupes_vs_prev_table failed: %s", e)
3328
  return text
3329
 
3330
 
 
3386
  return new
3387
 
3388
  text = _UCB_DEP_CONT_BLOCK.sub(_repl, text)
3389
+ # Fix 17e: repeat fused EC/ED fix + duplicate ED strip + 17g prune until stable.
3390
  for _ in range(16):
3391
  nxt = _split_ucb_fused_ec_ed_headings_single_table_html(text)
3392
  nxt = _ucb_strip_redundant_ed_subset_fragment(nxt)
3393
+ nxt = _ucb_prune_ed_fragment_dupes_vs_prev_table(nxt)
3394
  if nxt == text:
3395
  break
3396
  text = nxt
 
3602
  """
3603
  Fix 30: Drop small DA3 table fragments (1-5 data rows) that are a loose
3604
  subset of an earlier larger DA3 table, matched by (date, amount) key alone.
3605
+
3606
+ This catches OCR-truncated duplicate rows where description text differs
3607
+ between the fragment and the anchor table, preventing the exact-key dedup
3608
+ in _dedupe_subset_transaction_tables_html from firing.
3609
+
3610
+ Example: BoA service-fee page — VIDDYOZE -1.11 appears in a 1-row
3611
+ fragment table after a full 2-row service-fee table that has the same
3612
+ row with a much longer description string.
3613
+
3614
+ Guards:
3615
+ - Fragment must have 1-5 data rows.
3616
+ - Anchor table must have more data rows than the fragment.
3617
+ - Every (date, amount) pair in the fragment must be covered by the anchor.
3618
+ - Date must be non-empty; amount must look like currency.
3619
+ - Wrapped in try/except (safe no-op on any error).
3620
  """
3621
  if not text or "<table" not in text.lower():
3622
  return text
 
3630
  _AMT_NONEMPTY = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?$")
3631
 
3632
  def _da_key(cells):
3633
+ """Loose (date_norm, amount_norm) key — description ignored."""
3634
  parts = [str(c or "").strip() for c in cells[:3]]
3635
  while len(parts) < 3:
3636
  parts.append("")
 
3639
  amt_s = amt_s.replace("\u2212", "-").replace("\u2013", "-")
3640
  return (date_s, amt_s)
3641
 
3642
+ # Parse all DA3 tables and collect their (date, amount) key lists.
3643
  da3_info = []
3644
  for m in matches:
3645
  g = _parse_da3_grid_from_table_html(m.group(0))
 
3673
  if tj["n"] <= ti["n"]:
3674
  continue
3675
  big_keys = Counter(tj["keys"])
3676
+ # Every (date, amount) pair in the fragment must be covered
3677
+ # by the anchor table (multiset subset check).
3678
  if all(frag_keys[k] <= big_keys.get(k, 0) for k in frag_keys):
3679
  drop_set.add(ti["match"].start())
3680
  log.info(
 
3975
  """
3976
  Fix 28: When OCR prepends the rows of a preceding credits/deposits DA3 table
3977
  onto the top of a debits/withdrawals DA3 table, strip that prefix.
3978
+ Safety guards (ALL must pass):
3979
+ 1. Both tables must be valid DA3 (Date|Description|Amount header).
3980
+ 2. Target (later) table must follow a debit/withdrawal heading.
3981
+ 3. Source (earlier) table must follow a credit/deposit heading.
3982
+ 4. Only the contiguous leading prefix of matching rows is removed.
3983
+ 5. At least 1 data row must remain in the target after stripping.
3984
+ 6. Strip count must be >= 1.
3985
+ 7. Entire function wrapped in try/except — returns input unchanged on error.
3986
  """
3987
  if not text or "<table" not in text.lower():
3988
  return text
 
4174
  if not page_md:
4175
  return page_md
4176
 
4177
+ # Fresh page: drop any text-layer dedupe caps so the first normalize pass
4178
+ # (before _patch_ocr_with_textlayer) does not see stale keys from a prior page.
4179
  _tl_freq_context.clear()
4180
 
4181
  page_md = normalize_money_glyphs(page_md)
 
4293
 
4294
  all_pages = []
4295
  for page_num, page_result in enumerate(results):
4296
+ # Header normalize runs before body patch; do not reuse prior page caps.
4297
  _tl_freq_context.clear()
4298
  page_md, regions = get_page_md_and_regions(page_result)
4299
  img_h = page_heights[page_num] if page_num < len(page_heights) else 1000