rehan953 commited on
Commit
a47bd8f
·
verified ·
1 Parent(s): adf086c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +179 -623
app.py CHANGED
@@ -41,20 +41,24 @@ Primary goals for reconcile rate:
41
  - Fix 21: Cross-table DA3 repair for MSBILL / MICROSOFT#G… PIN rows — when Amount is junk in one table
42
  but another table repeats the same authorization id with a strict amount, copy that amount
43
  (then subset dedupe can drop the stray duplicate block, e.g. under Deposits).
44
- - Fix 22: Drop a single OCR flat line between pages: (continued) + DESCRIPTION + AMOUNT + many MM/DD
45
  tokens (noise before a proper <table>).
46
  - Fix 23: Doc-wide DA3 table dedupe using multiset of (date, amount) keys — catches duplicate sections when
47
  OCR varies description text between copies (e.g. withdrawals repeated under Deposits).
48
  - Fix 24: Run orphan flatline strip on merged PDF output and after text-layer patch so page-boundary
49
  noise is removed consistently.
50
- - Fix 25: After a centered deposits + credits/interest heading, if two DA3 tables appear back-to-back
51
  and the first is mostly credit/deposit rows while the second is mostly debit/withdrawal-style
52
  rows (generic keywords), drop the second — fixes OCR pasting the withdrawals block again.
53
  - Fix 23b: Loose multiset keys use normalized float amounts so comma/spacing variants still match.
54
  - Fix 26: Drop small DA3 fragments (3–14 rows) that are a multiset subset of an earlier larger DA3
55
  table — matches PDF layouts that re-print a header and repeat a few rows.
56
- - Fix 27: Strip a lone …(continued) account banner line (no DATE/DESCRIPTION/AMOUNT) between
57
  page separators — duplicate header noise without the long flatline.
 
 
 
 
58
  """
59
 
60
  # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -612,10 +616,6 @@ _HEADER_ROW_KEYWORD_THRESHOLD = 0.5
612
  # --------------------------
613
 
614
  def _split_keyword_remainder(cell_text: str):
615
- """
616
- If cell_text starts with a known header keyword followed by extra content,
617
- return (keyword, remainder). Otherwise return (cell_text, "").
618
- """
619
  s = cell_text.strip()
620
  for pattern in _HEADER_KW_RES:
621
  m = pattern.match(s)
@@ -626,18 +626,6 @@ def _split_keyword_remainder(cell_text: str):
626
  return s, ""
627
 
628
  def _extract_fused_header_artifacts(header_row):
629
- """
630
- Generic detector for the OCR artifact where the first data row of a table
631
- gets fused into the header cells during OCR.
632
- The artifact pattern (BOTH signals must fire simultaneously):
633
- - Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
634
- e.g. "DESCRIPTIONBeginning Balance" "NARRATIONOpening Balance"
635
- - Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
636
- e.g. "BALANCE$3,447.10" "AMOUNT 5,000.00"
637
- Two-signal guard prevents false positives on clean PDFs from any bank.
638
- Returns:
639
- (cleaned_header : list[str], recovered_row : list[str] | None)
640
- """
641
  if not header_row:
642
  return list(header_row), None
643
 
@@ -676,10 +664,6 @@ def _extract_fused_header_artifacts(header_row):
676
  return list(header_row), None
677
 
678
  def _clean_header_artifacts(grid):
679
- """
680
- Entry point for Fix 1 called from normalize_html_tables.
681
- Returns (grid, recovered_row | None).
682
- """
683
  if not grid or not grid[0]:
684
  return grid, None
685
 
@@ -692,12 +676,6 @@ def _clean_header_artifacts(grid):
692
  # --------------------------
693
 
694
  def _row_keyword_score(row):
695
- """
696
- Return the fraction of non-empty cells in this row that are pure header
697
- keywords (e.g. DATE, DESCRIPTION, DEBIT, CREDIT, BALANCE).
698
- Also handles cells like "DATE DESCRIPTION" where two keywords are
699
- space-joined into one cell — these count as a keyword cell too.
700
- """
701
  non_empty = [str(c or "").strip() for c in row if str(c or "").strip()]
702
  if not non_empty:
703
  return 0.0
@@ -714,8 +692,6 @@ def _row_keyword_score(row):
714
 
715
  return keyword_hits / len(non_empty)
716
 
717
- # Pattern: header cell contains balance/account info — do NOT promote away from it.
718
- # Matches things like "Beginning Balance:", "Ending Balance:", account numbers, "$20.48$3.46"
719
  _BALANCE_INFO_RE = re.compile(
720
  r"(beginning|ending|opening|closing)\s+balance"
721
  r"|account\s*#?\s*\d{5,}"
@@ -725,39 +701,12 @@ _BALANCE_INFO_RE = re.compile(
725
  )
726
 
727
  def _header_contains_balance_info(header_row):
728
- """
729
- Return True if any cell in the header row contains balance or account
730
- information — indicating this IS a legitimate header row even if its
731
- keyword score is low (e.g. Citi's "208479667 | Beginning Balance:$20.48 | ...").
732
- """
733
  for cell in header_row:
734
  if _BALANCE_INFO_RE.search(str(cell or "")):
735
  return True
736
  return False
737
 
738
  def _promote_misplaced_header_row(grid):
739
- """
740
- Detects and fixes the OCR artifact where real column headers land in a
741
- <td> data row instead of the <th> header row.
742
- Two scenarios handled:
743
- Scenario A — Simple misplaced header (e.g. TD Bank):
744
- <th>ACCOUNT ACTIVITY</th> ← section title, not a real header
745
- <td>DATE DESCRIPTION</td> ... ← real column headers in a data row
746
- <td>01/05 ...</td> ← data
747
- → Promote the keyword row to header, discard the section title rows above.
748
- Scenario B — Metadata header + misplaced column headers (e.g. Citi page 1):
749
- <th>208479667</th> <th>Beginning Balance:$20.48 Ending Balance:$3.46</th>
750
- <td>Date Description</td> <td>Debits</td> <td>Credits</td> <td>Balance</td>
751
- <td>04/01 DEBIT CARD... 8.01</td> ...
752
- → The existing th row has balance/account metadata (NOT a real column header).
753
- → Promote the keyword data row to header.
754
- → Preserve metadata row cells as a plain-text prefix OUTSIDE the table
755
- by embedding them as a leading data row with colspan (kept for reference).
756
- → Split any "Date Description" fused cells in data rows.
757
- Guard conditions (no-op if not met — safe for all other PDFs):
758
- - Current header keyword score < threshold.
759
- - A data row within the first 5 rows has keyword score >= threshold.
760
- """
761
  if not grid or len(grid) < 2:
762
  return grid
763
 
@@ -765,7 +714,6 @@ def _promote_misplaced_header_row(grid):
765
  if current_header_score >= _HEADER_ROW_KEYWORD_THRESHOLD:
766
  return grid
767
 
768
- # Search the first few data rows for a candidate keyword header row.
769
  candidate_idx = None
770
  candidate_score = 0.0
771
  for i in range(1, min(len(grid), 6)):
@@ -779,7 +727,6 @@ def _promote_misplaced_header_row(grid):
779
 
780
  candidate_row = grid[candidate_idx]
781
 
782
- # Expand any merged keyword cells (e.g. "DATE DESCRIPTION" → ["DATE", "DESCRIPTION"])
783
  expanded_header = []
784
  for cell in candidate_row:
785
  cell_s = str(cell or "").strip()
@@ -792,9 +739,6 @@ def _promote_misplaced_header_row(grid):
792
  new_ncols = len(expanded_header)
793
  col_expansion = new_ncols - len(candidate_row)
794
 
795
- # If the existing header row contains balance/account metadata (Scenario B),
796
- # preserve it as the first data row so the information is not lost.
797
- # This row will have its content in col 0 (joined) and blanks elsewhere.
798
  metadata_row = None
799
  if _header_contains_balance_info(grid[0]):
800
  meta_cells = [str(c or "").strip() for c in grid[0]]
@@ -802,7 +746,6 @@ def _promote_misplaced_header_row(grid):
802
  if meta_text:
803
  metadata_row = [meta_text] + [""] * (new_ncols - 1)
804
 
805
- # Re-align data rows below the candidate header
806
  new_data_rows = []
807
  for row in grid[candidate_idx + 1:]:
808
  if col_expansion > 0 and row:
@@ -823,7 +766,6 @@ def _promote_misplaced_header_row(grid):
823
  return r[:n]
824
 
825
  new_grid = [pad(expanded_header, new_ncols)]
826
- # Insert metadata row first if present (preserves Beginning/Ending Balance)
827
  if metadata_row is not None:
828
  new_grid.append(pad(metadata_row, new_ncols))
829
  for row in new_data_rows:
@@ -835,39 +777,16 @@ def _promote_misplaced_header_row(grid):
835
  # Fix 3: Fused key-value rows in summary sections
836
  # --------------------------
837
 
838
- # Matches a value suffix fused onto a label with no space.
839
- # e.g. "Beginning Interest Rate0.00%" → label + "0.00%"
840
- # "Number of days in this Period31" → label + "31"
841
- # Strict: no \s* between groups so label trailing-space detects space-separated values
842
  _FUSED_KV_RE = re.compile(
843
  r"^(.+?)(-?\d{1,3}(?:,\d{3})*(?:\.\d+)?%?)$",
844
  re.DOTALL,
845
  )
846
 
847
  def _fix_fused_keyvalue_rows(grid):
848
- """
849
- Fix rows where label+value are fused into the first cell with all other
850
- cells empty — common in Interest Summary / Fee Summary sections appended
851
- to a transaction table by OCR.
852
- e.g. "Beginning Interest Rate0.00%" → label="Beginning Interest Rate" value="0.00%"
853
- "Number of days in this Period31" → label="..." value="31"
854
- Guard conditions (no-op if not met — safe for all other tables):
855
- - All cells except first must be empty.
856
- - A non-space character must immediately precede the digit run (fused).
857
- - Both label and value parts must be non-empty after splitting.
858
- - The cell must NOT contain an account number (7+ digit run) — prevents
859
- incorrectly splitting "STREAMLINED CHECKING #208479667" or
860
- "Charges debited from account #208479667".
861
- - The cell must NOT contain a proper dollar amount with decimal — prevents
862
- incorrectly splitting metadata like "Beginning Balance:$20.48$3.46".
863
- """
864
  if not grid or len(grid) < 2:
865
  return grid
866
 
867
- # Matches a proper dollar/currency amount with decimal point — these appear
868
- # in legitimate label cells and must not trigger the fused-KV split.
869
  _DOLLAR_AMOUNT_RE = re.compile(r"\$\s*\d{1,3}(?:,\d{3})*\.\d{2}")
870
- # Matches a long account/reference number (7+ consecutive digits)
871
  _LONG_NUMBER_RE = re.compile(r"\d{7,}")
872
 
873
  ncols = len(grid[0])
@@ -886,21 +805,14 @@ def _fix_fused_keyvalue_rows(grid):
886
  new_grid.append(row)
887
  continue
888
 
889
- # Guard: skip rows containing a proper dollar amount with decimal
890
- # (these are metadata/balance rows, not fused key-value summary rows)
891
  if _DOLLAR_AMOUNT_RE.search(first):
892
  new_grid.append(row)
893
  continue
894
 
895
- # Guard: skip rows containing a long account/reference number (7+ digits)
896
- # (e.g. "STREAMLINED CHECKING #208479667", "account #208479667")
897
  if _LONG_NUMBER_RE.search(first):
898
  new_grid.append(row)
899
  continue
900
 
901
- # Guard: skip rows containing a # followed by digits or X-patterns
902
- # (card/account number labels like "Card account # XXXX XXXX XXXX 2889")
903
- # These are separator rows reconstructed by Fix5, not fused KV rows.
904
  if re.search(r"#\s*[\dX]", first):
905
  new_grid.append(row)
906
  continue
@@ -910,10 +822,9 @@ def _fix_fused_keyvalue_rows(grid):
910
  new_grid.append(row)
911
  continue
912
 
913
- label_raw = m.group(1) # trailing space means value was space-separated, not fused
914
  value = m.group(2).strip()
915
 
916
- # Space before value means NOT fused — "Interest Rate 0.00%" is clean
917
  if label_raw != label_raw.rstrip():
918
  new_grid.append(row)
919
  continue
@@ -936,16 +847,6 @@ def _fix_fused_keyvalue_rows(grid):
936
  # --------------------------
937
 
938
  def _extract_textlayer_rows(pdf_path: str, page_num: int):
939
- """
940
- Extract structured transaction rows from the PDF text layer using pymupdf.
941
- Returns list of {"date": str, "desc": str, "amount": str} dicts, or []
942
- if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
943
- transaction rows are found (guards against non-transaction pages).
944
- Supports multiple date formats:
945
- - Numeric: MM/DD, MM/DD/YY, MM/DD/YYYY (Chase, TD, BoA, Citi)
946
- - Month-name: Jan 16, Feb 7, Mar 05 (Capital One, Amex)
947
- - ISO: YYYY-MM-DD
948
- """
949
  try:
950
  import pymupdf as fitz
951
 
@@ -957,7 +858,6 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
957
  if not words:
958
  return []
959
 
960
- # Group words by y-bucket (5pt tolerance)
961
  lines_by_y = {}
962
  for w in words:
963
  x0, y0, word = float(w[0]), float(w[1]), w[4]
@@ -975,24 +875,21 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
975
  line_words = sorted(lines_by_y[y], key=lambda t: t[0])
976
  sorted_lines.append([w for _, w in line_words])
977
 
978
- # Date patterns — numeric (MM/DD, MM-DD variants) or month-name (Jan 16)
979
  _MONTHS = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
980
  date_re = re.compile(
981
- r"^\d{1,2}/\d{2}(?:/\d{2,4})?$" # MM/DD, MM/DD/YY, MM/DD/YYYY
982
- r"|^\d{1,2}-\d{2}(?:-\d{2,4})?$" # MM-DD, MM-DD-YY (East West Bank)
983
- r"|^\d{4}-\d{2}-\d{2}$" # YYYY-MM-DD
984
- r"|^" + _MONTHS + r"$", # "Jan", "Feb" etc (month-name date part 1)
985
  re.IGNORECASE,
986
  )
987
- # When a month-name date appears, the next token is the day number
988
  month_re = re.compile(r"^" + _MONTHS + r"$", re.IGNORECASE)
989
  day_re = re.compile(r"^\d{1,2}$")
990
 
991
- # Amount: -1,234.56 $193.63 -$240.46 - $500.00 (with space after -)
992
  amount_re = re.compile(
993
  r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
994
  r"|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
995
- r"|^-\s+\$\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$" # "- $500.00"
996
  )
997
 
998
  rows = []
@@ -1000,17 +897,14 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
1000
  if len(line) < 2:
1001
  continue
1002
 
1003
- # Detect date and find where description starts
1004
  date_str = None
1005
  desc_start = None
1006
 
1007
  if date_re.match(line[0]):
1008
  if month_re.match(line[0]) and len(line) > 1 and day_re.match(line[1]):
1009
- # Month-name date: "Jan 16" → two tokens
1010
  date_str = line[0] + " " + line[1]
1011
  desc_start = 2
1012
  else:
1013
- # Single-token numeric date
1014
  date_str = line[0]
1015
  desc_start = 1
1016
  else:
@@ -1019,11 +913,7 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
1019
  if desc_start is None or desc_start >= len(line):
1020
  continue
1021
 
1022
- # For Capital One: Trans Date and Post Date are both present
1023
- # Line looks like: Jan 16 Jan 16 CAPITAL ONE MOBILE PYMT - $500.00
1024
- # After consuming first date, check if next tokens are also a date
1025
  remaining = line[desc_start:]
1026
- # Capture post_date if next tokens are also a date
1027
  post_date_str = ""
1028
  if remaining and month_re.match(remaining[0]):
1029
  if len(remaining) > 1 and day_re.match(remaining[1]):
@@ -1038,7 +928,6 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
1038
  if len(remaining) < 2:
1039
  continue
1040
 
1041
- # Handle "- $500.00" split as two tokens at end
1042
  if (len(remaining) >= 2
1043
  and remaining[-2] == "-"
1044
  and remaining[-1].startswith("$")):
@@ -1055,16 +944,11 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
1055
  if not desc:
1056
  continue
1057
 
1058
- # Guard: description must contain at least one letter.
1059
- # Rows where description is purely digits/dates/amounts are
1060
- # DAILY BALANCE rows (e.g. "170,198.04 01-13 45,442.31 01-24"),
1061
- # not real transactions — skip them.
1062
  if not re.search(r"[A-Za-z]", desc):
1063
  continue
1064
 
1065
  rows.append({"date": date_str, "post_date": post_date_str, "desc": desc, "amount": amount_candidate})
1066
 
1067
- # Guard: require at least 2 rows to avoid false positives on non-transaction pages
1068
  return rows if len(rows) >= 2 else []
1069
 
1070
  except Exception as e:
@@ -1072,11 +956,6 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
1072
  return []
1073
 
1074
  def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1075
- """
1076
- Johnson Bank: when pdfminer column extraction fails (e.g. pymupdf line text or
1077
- multi-column layout), extract (date, check#, amount) triplets from the Checks
1078
- subsection. Anchored by 'Checks (' ... 'Daily Account Balance' only.
1079
- """
1080
  if not text_layer:
1081
  return []
1082
  m = re.search(
@@ -1105,16 +984,6 @@ def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1105
  return out
1106
 
1107
  def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
1108
- """
1109
- Parse Johnson Bank Checks and Daily Account Balance from pdfminer column-separated
1110
- page text and return as HTML tables.
1111
- pdfminer outputs multi-column sections as separate column lists:
1112
- Date Number Amount
1113
- 04-14 3259 45.72
1114
- becomes:
1115
- "Date\n04-14\n04-09\n\nNumber\n3259\n3260\n\nAmount\n45.72\n1,628.00"
1116
- Uses pdfminer (always available) not pymupdf.
1117
- """
1118
  if not text_layer:
1119
  return ""
1120
 
@@ -1125,7 +994,6 @@ def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, nee
1125
  lines = [l.strip() for l in text_layer.splitlines()]
1126
  output = []
1127
 
1128
- # ── Checks ───────────────────────────────────────────────────────────
1129
  if needs_checks:
1130
  checks = []
1131
  i = 0
@@ -1183,7 +1051,6 @@ def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, nee
1183
  + rows + "\n</table>"
1184
  )
1185
 
1186
- # ── Daily Account Balance ─────────────────────────────────────────────
1187
  if needs_dab:
1188
  balances = []
1189
  i = 0
@@ -1228,30 +1095,17 @@ def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, nee
1228
  return "\n\n".join(output)
1229
 
1230
  def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str:
1231
- """
1232
- Compare OCR table rows against PDF text-layer rows and inject any rows
1233
- the OCR model silently dropped (typically identical consecutive rows).
1234
- Guard conditions (all must pass for any injection):
1235
- 1. PDF text layer must exist and yield >= 2 transaction rows.
1236
- 2. The OCR table must have DATE + DESCRIPTION + AMOUNT columns.
1237
- 3. At least one (date, desc, amount) key must appear in both OCR and
1238
- text layer — ensures we are patching the right table.
1239
- 4. Only injects copies of rows ALREADY present in OCR (ocr_count > 0)
1240
- — never invents new row content.
1241
- 5. Entire function wrapped in try/except — any error returns page_md unchanged.
1242
- """
1243
  if not page_md or "<table" not in page_md.lower():
1244
  return page_md
1245
 
1246
  try:
1247
  tl_rows = _extract_textlayer_rows(pdf_path, page_num)
1248
  if not tl_rows:
1249
- return page_md # scanned PDF or non-transaction page — safe no-op
1250
 
1251
  def _norm(s):
1252
  return re.sub(r"\s+", " ", str(s or "").strip().lower())
1253
 
1254
- # Build text-layer frequency map
1255
  tl_freq = {}
1256
  for r in tl_rows:
1257
  key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
@@ -1269,7 +1123,6 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1269
  if len(grid) < 2:
1270
  continue
1271
 
1272
- # Identify DATE, DESCRIPTION, AMOUNT column indices
1273
  header = [str(c or "").strip().upper() for c in grid[0]]
1274
  date_col = desc_col = amt_col = None
1275
  for ci, h in enumerate(header):
@@ -1281,9 +1134,8 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1281
  amt_col = ci
1282
 
1283
  if date_col is None or desc_col is None or amt_col is None:
1284
- continue # not a transaction table
1285
 
1286
- # Build OCR frequency map
1287
  ocr_freq = {}
1288
  ocr_rows_indexed = []
1289
  for ri, row in enumerate(grid[1:], start=1):
@@ -1299,16 +1151,8 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1299
  ocr_freq[key] = ocr_freq.get(key, 0) + 1
1300
  ocr_rows_indexed.append((ri, key))
1301
 
1302
- # Guard: overlap check for Case A (duplicate restore).
1303
- # For Case A we require at least one row in both OCR and text layer
1304
- # to confirm we are patching the right table.
1305
- # For Case B (entirely missing rows) we use a lighter structural check:
1306
- # if the OCR table has DATE+DESCRIPTION+AMOUNT columns (already verified)
1307
- # and the text layer date format matches the OCR date format,
1308
- # we can safely inject — even when zero rows overlap.
1309
  overlap = set(ocr_freq.keys()) & set(tl_freq.keys())
1310
 
1311
- # Detect date format used in OCR table (MM/DD vs MM-DD vs Mon DD)
1312
  _ocr_dates = [
1313
  _norm(str(row[date_col] or ""))
1314
  for row in grid[1:]
@@ -1325,9 +1169,6 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1325
  tl_fmt = _date_fmt(_tl_dates)
1326
  date_fmt_match = (ocr_fmt == tl_fmt) or "other" in (ocr_fmt, tl_fmt)
1327
 
1328
- # Determine missing rows.
1329
- # Case A: row exists in OCR but count is too low (duplicate dropped)
1330
- # Case B: row exists in text layer but completely absent from OCR
1331
  to_inject = {}
1332
  to_inject_new = {}
1333
 
@@ -1337,12 +1178,9 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1337
  if tl_count > ocr_count:
1338
  missing = tl_count - ocr_count
1339
  if ocr_count > 0:
1340
- # Case A: restore duplicates — requires overlap confirmation
1341
  if overlap:
1342
  to_inject[key] = missing
1343
  else:
1344
- # Case B: entirely new row — requires date format match
1345
- # (lighter guard: no overlap needed, just structural match)
1346
  if date_fmt_match:
1347
  tl_row_data = next(
1348
  (r for r in tl_rows
@@ -1355,7 +1193,6 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1355
  if not to_inject and not to_inject_new:
1356
  continue
1357
 
1358
- # Build patched grid — inject missing duplicate rows (Case A)
1359
  new_grid = [grid[0]]
1360
  for ri, row in enumerate(grid[1:], start=1):
1361
  new_grid.append(row)
@@ -1376,15 +1213,10 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1376
  new_table_html = _grid_to_html(new_grid)
1377
  result = result.replace(table_html, new_table_html, 1)
1378
 
1379
- # Case B: completely missing rows → build a SEPARATE new table
1380
- # appended after the patched OCR table. Using a separate table
1381
- # preserves the original PDF structure (e.g. Capital One has
1382
- # separate "Payments" and "Transactions" sections).
1383
  if to_inject_new:
1384
  ncols = len(grid[0])
1385
  header_row = grid[0]
1386
 
1387
- # Find post_date column index if it exists in the header
1388
  post_date_col = None
1389
  for ci, h in enumerate(header_row):
1390
  hh = str(h or "").strip().upper()
@@ -1410,14 +1242,9 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1410
  if new_rows:
1411
  extra_grid = [header_row] + new_rows
1412
  extra_html = "\n\n" + _grid_to_html(extra_grid)
1413
- # Insert immediately after the patched table
1414
  insert_pos = result.find(new_table_html) + len(new_table_html)
1415
  result = result[:insert_pos] + extra_html + result[insert_pos:]
1416
 
1417
- # Case C: page has text-layer rows but NO transaction table in OCR output.
1418
- # Build a new table from the text layer and prepend it.
1419
- # Guard: only fires when the page has NO table with both DATE + DESCRIPTION cols.
1420
- # This is placed AFTER the for-loop so it only runs once per page, not per table.
1421
  if tl_rows:
1422
  _has_txn_table = False
1423
  for _tbl in table_pattern.finditer(result):
@@ -1457,44 +1284,18 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1457
 
1458
  except Exception as e:
1459
  log.warning("_patch_ocr_with_textlayer failed (page %d): %s", page_num, e)
1460
- return page_md # always safe — return original on any error
1461
 
1462
  # --------------------------
1463
  # Fix 5: Mid-table separator row reconstruction
1464
  # --------------------------
1465
 
1466
- # Date pattern for transaction rows: MM/DD, MM/DD/YY, MM/DD/YYYY
1467
  _DATE_RE = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
1468
-
1469
- # Standalone money amount (may be split off a separator label by OCR)
1470
  _SPLIT_AMOUNT_RE = re.compile(
1471
  r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
1472
  )
1473
 
1474
  def _reconstruct_separator_rows(grid):
1475
- """
1476
- Detect and reconstruct mid-table separator / label rows that OCR has
1477
- incorrectly split across columns.
1478
- The artifact (Bank of America and similar):
1479
- A section label like "Card account # XXXX XXXX XXXX 2889" sits between
1480
- transaction rows as a full-width label. OCR splits the trailing digits
1481
- across columns because they are right-aligned, producing variations like:
1482
- 2-cell split: ["Card account # XXXX XXXX XXXX 2", "", "889"]
1483
- 3-cell split: ["Card account # XXXX XXXX XXXX", "2", "889"]
1484
- clean 1-cell: ["Card account # XXXX XXXX XXXX 2889", "", ""]
1485
- Detection criteria (ALL must hold — guards safe for all other PDFs):
1486
- 1. First cell is NOT a date token (transaction rows always start with date).
1487
- 2. First cell contains letters (it is a label, not a bare number).
1488
- 3. All cells except the first either:
1489
- a. are empty, OR
1490
- b. are a short pure-digit fragment (1-4 digits, no decimal, no sign)
1491
- — these are the split-off tails of the account/card number.
1492
- 4. There must be at least one non-empty cell after the first (to detect
1493
- the split; single-cell rows are also handled as clean label rows).
1494
- The fix:
1495
- Concatenate all non-empty cells in order, place the result in col 0,
1496
- blank all other cells.
1497
- """
1498
  if not grid or len(grid) < 2:
1499
  return grid
1500
 
@@ -1502,7 +1303,7 @@ def _reconstruct_separator_rows(grid):
1502
  if ncols < 2:
1503
  return grid
1504
 
1505
- new_grid = [grid[0]] # keep header unchanged
1506
 
1507
  for row in grid[1:]:
1508
  cells = [str(c or "").strip() for c in row]
@@ -1511,36 +1312,26 @@ def _reconstruct_separator_rows(grid):
1511
 
1512
  non_empty = [(i, c) for i, c in enumerate(cells) if c]
1513
 
1514
- # Completely empty row — leave as-is
1515
  if len(non_empty) == 0:
1516
  new_grid.append(row)
1517
  continue
1518
 
1519
  first_idx, first_val = non_empty[0]
1520
 
1521
- # Guard 1: first cell must not be a date token
1522
  if _DATE_RE.match(first_val):
1523
  new_grid.append(row)
1524
  continue
1525
 
1526
- # Guard 2: first cell must contain letters (label, not a bare number)
1527
  if not re.search(r"[A-Za-z]", first_val):
1528
  new_grid.append(row)
1529
  continue
1530
 
1531
- # Single non-empty cell — already a clean label row
1532
  if len(non_empty) == 1:
1533
  new_row = [""] * ncols
1534
  new_row[0] = first_val
1535
  new_grid.append(new_row)
1536
  continue
1537
 
1538
- # Multiple non-empty cells: check that every cell AFTER the first
1539
- # is a short pure-digit fragment (1-4 digits, no decimal, no sign).
1540
- # This is the key generic guard — it allows 2, 3, or more cells
1541
- # as long as all the trailing cells are digit-only fragments.
1542
- # Real transaction rows always have amounts with decimals (-14.19)
1543
- # or descriptions with letters, so they never pass this check.
1544
  trailing = [val for _, val in non_empty[1:]]
1545
  all_trailing_are_digit_fragments = all(
1546
  re.fullmatch(r"\d{1,4}", v) for v in trailing
@@ -1550,7 +1341,6 @@ def _reconstruct_separator_rows(grid):
1550
  new_grid.append(row)
1551
  continue
1552
 
1553
- # Reconstruct: concatenate all non-empty cells in order
1554
  reconstructed = "".join(val for _, val in non_empty).strip()
1555
  new_row = [""] * ncols
1556
  new_row[0] = reconstructed
@@ -1563,27 +1353,6 @@ def _reconstruct_separator_rows(grid):
1563
  # --------------------------
1564
 
1565
  def _merge_split_rows(grid):
1566
- """
1567
- Fix OCR artifact where a transaction row is split across two <tr> rows
1568
- because the description wrapped to a second line in the source PDF.
1569
- Two patterns detected (Hardin County Bank and similar monospace statements):
1570
- Pattern A — description + empty continuation:
1571
- Row N: [desc, '', '', '', ''] ← description, no date/money
1572
- Row N+1: ['', debit, credit, date, bal] ← money/date, no description
1573
- → Merge into: [desc, debit, credit, date, bal]
1574
- Pattern B — description + text continuation + money:
1575
- Row N: [desc, '', '', '', ''] ← first line of description
1576
- Row N+1: [desc_cont, debit, credit, date, bal] ← continuation + money
1577
- → Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
1578
- Guard conditions (no-op unless both rows match the pattern):
1579
- - Row N must have non-empty col 0 (description).
1580
- - Row N must have empty date column (col 3 or wherever DATE is).
1581
- - Row N must have empty debit AND credit columns.
1582
- - Row N+1 must have non-empty date column.
1583
- - Row N+1 must have non-empty debit OR credit column.
1584
- - Row N balance (if present) must be a short fragment without a decimal
1585
- (1-6 chars, no '.') — confirms it's a split-off ref number, not a balance.
1586
- """
1587
  if not grid or len(grid) < 3:
1588
  return grid
1589
 
@@ -1591,7 +1360,6 @@ def _merge_split_rows(grid):
1591
  if ncols < 3:
1592
  return grid
1593
 
1594
- # Identify column indices from header
1595
  header = [str(c or "").strip().upper() for c in grid[0]]
1596
  date_col = desc_col = bal_col = None
1597
  debit_cols = []
@@ -1609,7 +1377,6 @@ def _merge_split_rows(grid):
1609
  if re.search(r"\b(CREDIT|CREDITS|CR|DEPOSIT|ADDITIONS?)\b", h):
1610
  credit_cols.append(ci)
1611
 
1612
- # Need at least date + description + one money column
1613
  if date_col is None or desc_col is None:
1614
  return grid
1615
  if not debit_cols and not credit_cols:
@@ -1618,7 +1385,6 @@ def _merge_split_rows(grid):
1618
  money_cols = debit_cols + credit_cols
1619
  _MONEY_RE2 = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*\.\d{1,4}$")
1620
  _DATE_RE2 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$|^\d{1,2}-\d{2}(?:-\d{2,4})?$")
1621
- # Short fragment guard: 1-6 non-space chars, no decimal point → split-off ref tail
1622
  _FRAGMENT_RE = re.compile(r"^[^\s\.]{1,6}$")
1623
 
1624
  new_grid = [grid[0]]
@@ -1628,7 +1394,6 @@ def _merge_split_rows(grid):
1628
  while len(row) < ncols:
1629
  row.append("")
1630
 
1631
- # Check if this row is a "description-only" split row candidate
1632
  has_desc = bool(row[desc_col])
1633
  no_date = not row[date_col]
1634
  no_money = all(not row[c] for c in money_cols)
@@ -1644,17 +1409,13 @@ def _merge_split_rows(grid):
1644
  next_has_money = any(bool(next_row[c]) for c in money_cols)
1645
 
1646
  if next_has_date and next_has_money:
1647
- # Pattern A or B — merge
1648
  merged = list(next_row)
1649
- # Append description (and possible continuation from next row)
1650
  if next_row[desc_col]:
1651
- # Pattern B: next row has desc continuation
1652
  merged[desc_col] = row[desc_col] + " " + next_row[desc_col]
1653
  else:
1654
- # Pattern A: next row has no description
1655
  merged[desc_col] = row[desc_col]
1656
  new_grid.append(merged)
1657
- i += 2 # skip both rows, emit merged
1658
  continue
1659
 
1660
  new_grid.append(row)
@@ -1663,17 +1424,6 @@ def _merge_split_rows(grid):
1663
  return new_grid
1664
 
1665
  def _extract_fused_desc_amount(grid):
1666
- """
1667
- Fix OCR artifact where a trailing money amount gets fused into the
1668
- description cell instead of its own debit/credit column.
1669
- Example (Hardin County Bank):
1670
- OCR: ['CHASE CREDIT CRD EPAY 8319249882 500.00', '', '', '04/11/25', '16,699.04']
1671
- Fix: ['CHASE CREDIT CRD EPAY 8319249882', '500.00', '', '04/11/25', '16,699.04']
1672
- Guard conditions (all must hold):
1673
- - Description ends with a space + money amount (NNN.NN or N,NNN.NN).
1674
- - ALL debit AND credit columns are empty for this row.
1675
- - Date column is non-empty (confirms it is a complete data row).
1676
- """
1677
  if not grid or len(grid) < 2:
1678
  return grid
1679
 
@@ -1724,15 +1474,6 @@ def _extract_fused_desc_amount(grid):
1724
  return new_grid
1725
 
1726
  def _is_junk_table(grid):
1727
- """
1728
- Detect and suppress known junk tables that are OCR artifacts from
1729
- page headers/footers, not real transaction data.
1730
- Current patterns:
1731
- 1. CUSTOMER INFORMATION fragment tables (First Horizon Bank):
1732
- Single-column table with header "CUSTOMER INFORMATION" and
1733
- data rows that are short numeric fragments (e.g. "254", "24")
1734
- from the partial account number / year printed in the header band.
1735
- """
1736
  if not grid or len(grid) < 2:
1737
  return False
1738
  ncols = len(grid[0])
@@ -1740,24 +1481,12 @@ def _is_junk_table(grid):
1740
  return False
1741
  header_text = str(grid[0][0] or "").strip().upper()
1742
  if "CUSTOMER INFORMATION" in header_text:
1743
- # All data rows should be short (< 10 chars) numeric/alphanumeric fragments
1744
  data_rows = grid[1:]
1745
  if all(len(str(r[0] or "").strip()) <= 10 for r in data_rows):
1746
  return True
1747
  return False
1748
 
1749
  def _split_fused_multicolumn_header(grid):
1750
- """
1751
- Fix OCR artifact where multiple column headers are fused into one <th> cell.
1752
- Example (First Horizon Bank alternating pages):
1753
- Fused: ['DATE DESCRIPTION CARD #', 'DEPOSIT', 'WITHDRAWAL']
1754
- Correct: ['DATE', 'DESCRIPTION', 'DEPOSIT', 'WITHDRAWAL', 'CARD #']
1755
- The data rows also have date + description + card# all in col 0:
1756
- Fused: ['01/16 PURCHASE - UBER TRIP... 4919', '', '$21.02']
1757
- Correct: ['01/16', 'PURCHASE - UBER TRIP...', '', '$21.02', '4919']
1758
- Detection: first header cell contains 2+ keyword terms separated by spaces,
1759
- AND remaining header cells are valid money column keywords.
1760
- """
1761
  if not grid or len(grid) < 2:
1762
  return grid
1763
 
@@ -1766,20 +1495,15 @@ def _split_fused_multicolumn_header(grid):
1766
  return grid
1767
 
1768
  first_header = str(grid[0][0] or "").strip()
1769
- # Split first header cell into parts and check if multiple are keywords
1770
  parts = first_header.split()
1771
  kw_hits = [p for p in parts if _HEADER_KW_EXACT_RE.match(p)]
1772
  if len(kw_hits) < 2:
1773
- return grid # not a fused multi-keyword header
1774
 
1775
- # The remaining header cells must all be keyword columns
1776
  rest_headers = [str(c or "").strip() for c in grid[0][1:]]
1777
  if not all(_HEADER_KW_EXACT_RE.match(h) for h in rest_headers if h):
1778
  return grid
1779
 
1780
- # Build the new expanded header:
1781
- # Split the fused first cell into individual keyword columns
1782
- # Keep non-keyword trailing parts (e.g. "CARD #") as the last new col
1783
  new_header_cols = []
1784
  non_kw_parts = []
1785
  for p in parts:
@@ -1793,12 +1517,10 @@ def _split_fused_multicolumn_header(grid):
1793
  if non_kw_parts:
1794
  new_header_cols.append(" ".join(non_kw_parts))
1795
 
1796
- # Full new header = expanded first cell + remaining cells
1797
  new_header = new_header_cols + rest_headers
1798
  new_ncols = len(new_header)
1799
- extra_cols = new_ncols - ncols # how many new columns were added
1800
 
1801
- # Identify column roles in the new header
1802
  _DATE_COL_RE = re.compile(r"\bDATE\b", re.IGNORECASE)
1803
  _CARD_COL_RE = re.compile(r"\bCARD\b", re.IGNORECASE)
1804
  _DESC_COL_RE = re.compile(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", re.IGNORECASE)
@@ -1809,11 +1531,10 @@ def _split_fused_multicolumn_header(grid):
1809
  new_card_col = next((i for i,h in enumerate(new_header) if _CARD_COL_RE.search(h)), None)
1810
 
1811
  if new_date_col is None or new_desc_col is None:
1812
- return grid # can't map columns
1813
 
1814
- # Re-split each data row: col 0 had "date description... card#" fused together
1815
  _DATE_PREFIX = re.compile(r"^(\d{1,2}/\d{2}(?:/\d{2,4})?)\s+(.*)", re.DOTALL)
1816
- _CARD_SUFFIX = re.compile(r"^(.*?)\s+(\d{4})$", re.DOTALL) # 4-digit card number at end
1817
 
1818
  def pad(row, n):
1819
  r = list(row)
@@ -1828,7 +1549,6 @@ def _split_fused_multicolumn_header(grid):
1828
  fused_cell = cells[0]
1829
  rest_cells = cells[1:]
1830
 
1831
- # Extract date from front
1832
  date_val = ""
1833
  desc_val = fused_cell
1834
  card_val = ""
@@ -1838,20 +1558,17 @@ def _split_fused_multicolumn_header(grid):
1838
  date_val = dm.group(1)
1839
  desc_val = dm.group(2).strip()
1840
 
1841
- # Extract card# from end (4-digit number)
1842
  if new_card_col is not None:
1843
  cm = _CARD_SUFFIX.match(desc_val)
1844
  if cm:
1845
  desc_val = cm.group(1).strip()
1846
  card_val = cm.group(2)
1847
 
1848
- # Build new row
1849
  new_row = [""] * new_ncols
1850
  new_row[new_date_col] = date_val
1851
  new_row[new_desc_col] = desc_val
1852
  if new_card_col is not None:
1853
  new_row[new_card_col] = card_val
1854
- # Fill remaining money columns from rest_cells
1855
  money_col_indices = [i for i,h in enumerate(new_header) if _MONEY_COL_RE.search(h)]
1856
  for mi, ri in enumerate(range(len(rest_cells))):
1857
  if mi < len(money_col_indices):
@@ -1862,17 +1579,6 @@ def _split_fused_multicolumn_header(grid):
1862
  return new_grid
1863
 
1864
  def _normalize_daily_balance_table(grid):
1865
- """
1866
- Fix OCR artifact in DAILY BALANCE SUMMARY tables where:
1867
- - OCR produces N DATE columns followed by N BALANCE columns
1868
- instead of interleaved DATE|BALANCE|DATE|BALANCE pairs
1869
- - Multiple dates are stacked in one cell "01/02\n01/03\n01/04\n01/05"
1870
- Reorders columns and expands stacked date cells into proper rows.
1871
- Only fires when ALL of these hold:
1872
- - Header has exactly 2N columns where N >= 2
1873
- - First N columns are all DATE, last N columns are all BALANCE
1874
- - At least one data cell contains a newline (stacked dates)
1875
- """
1876
  if not grid or len(grid) < 2:
1877
  return grid
1878
 
@@ -1891,13 +1597,11 @@ def _normalize_daily_balance_table(grid):
1891
  if not all(h == "BALANCE" for h in bal_half):
1892
  return grid
1893
 
1894
- # Build new interleaved header: DATE BALANCE DATE BALANCE ...
1895
  new_header = []
1896
  for i in range(half):
1897
  new_header.append("DATE")
1898
  new_header.append("BALANCE")
1899
 
1900
- # Expand each data row
1901
  new_rows = []
1902
  _DATE_RE3 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
1903
 
@@ -1906,23 +1610,17 @@ def _normalize_daily_balance_table(grid):
1906
  while len(cells) < ncols:
1907
  cells.append("")
1908
 
1909
- # Dates are all stacked in cells[0]; balances follow immediately in cells[1..N]
1910
- # (OCR layout: stacked_dates | bal1 | bal2 | bal3 | bal4 | empty...)
1911
- # NOT: date | date | date | date | bal | bal | bal | bal (despite header order)
1912
  date_cells = [cells[0]]
1913
  bal_cells = cells[1:]
1914
 
1915
- # Check if dates are stacked in first cell (newline OR space-separated)
1916
- # e.g. "01/02 01/03 01/04 01/05" or "01/02\n01/03\n01/04\n01/05"
1917
  first = date_cells[0]
1918
  tokens = re.split(r"[\n\r\s]+", first)
1919
  date_tokens = [t.strip() for t in tokens if t.strip() and _DATE_RE3.match(t.strip())]
1920
  if len(date_tokens) >= 2:
1921
  stacked = date_tokens
1922
  else:
1923
- stacked = date_cells # already one date per cell
1924
 
1925
- # Build ONE interleaved row for this group of stacked dates
1926
  new_row = [""] * ncols
1927
  for i, date_val in enumerate(stacked):
1928
  if not _DATE_RE3.match(date_val):
@@ -1938,16 +1636,6 @@ def _normalize_daily_balance_table(grid):
1938
  return [new_header] + new_rows
1939
 
1940
  def _normalize_checks_paid_table(grid):
1941
- """
1942
- Fix OCR artifact in CHECKS PAID SUMMARY tables where:
1943
- - Header col0 = "DATE CHECK # DATE CHECK # DATE CHECK #" (fused repeating groups)
1944
- - Remaining headers = "AMOUNT AMOUNT AMOUNT"
1945
- - Data col0 = "01/24 4701 01/18 4704 * 01/18 916831 *" (all groups fused)
1946
- - Remaining data cells = amount values
1947
- Expands to proper: DATE | CHECK # | AMOUNT | DATE | CHECK # | AMOUNT | ...
1948
- Detection: first header cell matches pattern (DATE CHECK #)+
1949
- and remaining headers are all AMOUNT.
1950
- """
1951
  if not grid or len(grid) < 2:
1952
  return grid
1953
 
@@ -1960,17 +1648,14 @@ def _normalize_checks_paid_table(grid):
1960
  if not _CHECKS_HDR.match(first_h):
1961
  return grid
1962
 
1963
- # Remaining headers must be AMOUNT (or empty)
1964
  rest_h = [str(c or "").strip().upper() for c in grid[0][1:]]
1965
  if not all(h in ("AMOUNT", "") for h in rest_h if h):
1966
  return grid
1967
 
1968
- # Count groups from how many times DATE appears in the fused header
1969
  n_groups = len(re.findall(r"\bDATE\b", first_h, re.IGNORECASE))
1970
  if n_groups < 1:
1971
  return grid
1972
 
1973
- # Build new header: DATE | CHECK # | AMOUNT repeated n_groups times
1974
  new_header = []
1975
  for _ in range(n_groups):
1976
  new_header.extend(["DATE", "CHECK #", "AMOUNT"])
@@ -1986,7 +1671,6 @@ def _normalize_checks_paid_table(grid):
1986
  fused_data = cells[0]
1987
  amounts = cells[1:]
1988
 
1989
- # Split fused data into groups by date boundary
1990
  tokens = fused_data.split()
1991
  groups = []
1992
  current = []
@@ -1999,31 +1683,18 @@ def _normalize_checks_paid_table(grid):
1999
  if current and current[0]:
2000
  groups.append(current)
2001
 
2002
- # Build one output row
2003
  new_row = [""] * len(new_header)
2004
  for i, grp in enumerate(groups):
2005
  if i >= n_groups:
2006
  break
2007
- new_row[i * 3] = grp[0] # DATE
2008
- new_row[i * 3 + 1] = " ".join(grp[1:]).strip() # CHECK #
2009
- new_row[i * 3 + 2] = amounts[i] if i < len(amounts) else "" # AMOUNT
2010
 
2011
  new_rows.append(new_row)
2012
 
2013
  return new_rows
2014
 
2015
- # ── Johnson Bank plain-text section converter ─────────────────────────────
2016
- # Johnson Bank formats Deposits, Withdrawals, Checks, and Daily Account Balance
2017
- # as plain text columns (no HTML tables). This converter detects those sections
2018
- # and emits proper <table> HTML.
2019
- #
2020
- # SAFETY GUARDS — all four must hold before any conversion fires:
2021
- # 1. Line matches exact section-header pattern
2022
- # 2. Next non-blank line matches exact column-header pattern
2023
- # 3. First data row uses MM-DD date format (Johnson Bank's unique date separator)
2024
- # 4. The candidate section contains NO existing <table> tags
2025
- # Guard 3 is the key discriminator: all other supported banks use MM/DD.
2026
-
2027
  _JB_SECTION_HDR = re.compile(
2028
  r"^(Deposits?|Withdrawals?|Checks?(?:\s+Paid)?|Daily\s+Account\s+Balance)"
2029
  r"\s*(?:\(cont\.?\))?\s*$",
@@ -2034,7 +1705,6 @@ _JB_COL_HDR = re.compile(
2034
  r"|^Date\s+(?:Number\s+)?(?:Balance|Amount)\s*$",
2035
  re.IGNORECASE,
2036
  )
2037
- # MM-DD date (dash separator) — Johnson Bank's unique format
2038
  _JB_DATE_DD = re.compile(r"^\d{2}-\d{2}\s+")
2039
  _JB_TXN_ROW = re.compile(r"^(\d{2}-\d{2})\s+(.+?)\s+(-?\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
2040
  _JB_BAL_ROW = re.compile(r"^(\d{2}-\d{2})\s+(\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
@@ -2049,24 +1719,15 @@ def _jb_rows_to_html(header_cols, rows):
2049
  return "\n".join(parts)
2050
 
2051
  def _jb_section_has_table(lines, start, end):
2052
- """Return True if any line in lines[start:end] contains a <table tag."""
2053
  return any("<table" in l.lower() for l in lines[start:end])
2054
 
2055
  def _find_first_data_row(lines, start):
2056
- """Return index of first non-blank line after start, or None."""
2057
  for i in range(start, min(start + 20, len(lines))):
2058
  if lines[i].strip():
2059
  return i
2060
  return None
2061
 
2062
  def convert_plaintext_bank_sections(text: str) -> str:
2063
- """
2064
- Convert Johnson Bank plain-text transaction sections to HTML tables.
2065
- All four safety guards must pass before conversion fires.
2066
- Safe no-op for all other banks.
2067
- """
2068
- # Quick exit: if the page already has lots of tables, skip entirely
2069
- # (handles pages that are already properly formatted)
2070
  lines = text.splitlines()
2071
  out = []
2072
  i = 0
@@ -2074,7 +1735,6 @@ def convert_plaintext_bank_sections(text: str) -> str:
2074
  while i < len(lines):
2075
  line = lines[i].strip()
2076
 
2077
- # Guard 1: section header
2078
  if not _JB_SECTION_HDR.match(line):
2079
  out.append(lines[i])
2080
  i += 1
@@ -2083,11 +1743,9 @@ def convert_plaintext_bank_sections(text: str) -> str:
2083
  section_title = line
2084
  j = i + 1
2085
 
2086
- # Skip blank lines to find column header
2087
  while j < len(lines) and not lines[j].strip():
2088
  j += 1
2089
 
2090
- # Guard 2: column header
2091
  if j >= len(lines) or not _JB_COL_HDR.match(lines[j].strip()):
2092
  out.append(lines[i])
2093
  i += 1
@@ -2096,16 +1754,13 @@ def convert_plaintext_bank_sections(text: str) -> str:
2096
  col_hdr_line = lines[j].strip()
2097
  data_start = j + 1
2098
 
2099
- # Find first non-blank data line
2100
  first_data_idx = _find_first_data_row(lines, data_start)
2101
 
2102
- # Guard 3: first data row must use MM-DD date format
2103
  if first_data_idx is None or not _JB_DATE_DD.match(lines[first_data_idx].strip()):
2104
  out.append(lines[i])
2105
  i += 1
2106
  continue
2107
 
2108
- # Find section end (next section header, page separator, or end of text)
2109
  section_end = len(lines)
2110
  for k in range(data_start, len(lines)):
2111
  l = lines[k].strip()
@@ -2115,13 +1770,11 @@ def convert_plaintext_bank_sections(text: str) -> str:
2115
  section_end = k
2116
  break
2117
 
2118
- # Guard 4: no existing <table> tags in this section
2119
  if _jb_section_has_table(lines, i, section_end):
2120
  out.append(lines[i])
2121
  i += 1
2122
  continue
2123
 
2124
- # All guards passed — parse and convert
2125
  parts_upper = col_hdr_line.upper().split()
2126
  if len(parts_upper) == 2 and parts_upper[1] == "BALANCE":
2127
  header_cols = ["Date", "Balance"]
@@ -2148,7 +1801,6 @@ def convert_plaintext_bank_sections(text: str) -> str:
2148
  or row_line.startswith("Page:")):
2149
  break
2150
 
2151
- # Balance-only row
2152
  mb = _JB_BAL_ROW.match(row_line)
2153
  if mb and header_cols == ["Date", "Balance"]:
2154
  if current:
@@ -2157,7 +1809,6 @@ def convert_plaintext_bank_sections(text: str) -> str:
2157
  k += 1
2158
  continue
2159
 
2160
- # Full transaction row
2161
  mt = _JB_TXN_ROW.match(row_line)
2162
  if mt:
2163
  if current:
@@ -2171,7 +1822,6 @@ def convert_plaintext_bank_sections(text: str) -> str:
2171
  k += 1
2172
  continue
2173
 
2174
- # Continuation line
2175
  if current and row_line:
2176
  desc_key = "Description" if "Description" in header_cols else "Number"
2177
  if desc_key in current:
@@ -2187,7 +1837,7 @@ def convert_plaintext_bank_sections(text: str) -> str:
2187
  if rows:
2188
  out.append(_jb_rows_to_html(header_cols, rows))
2189
 
2190
- i = section_end # jump past the whole section
2191
  continue
2192
 
2193
  return "\n".join(out)
@@ -2197,10 +1847,6 @@ def convert_plaintext_bank_sections(text: str) -> str:
2197
  # --------------------------
2198
 
2199
  def _is_nfcu_document(pdf_path: str) -> bool:
2200
- """
2201
- Strict detector for Navy Federal statements.
2202
- Used to gate NF-specific parsing so other banks are unaffected.
2203
- """
2204
  try:
2205
  import pymupdf as fitz
2206
  doc = fitz.open(pdf_path)
@@ -2210,7 +1856,6 @@ def _is_nfcu_document(pdf_path: str) -> bool:
2210
  text += "\n" + (doc[i].get_text() or "")
2211
  doc.close()
2212
  t = text.lower()
2213
- # Strict Navy-only guard to avoid affecting other statement types.
2214
  return (
2215
  ("statement of account" in t)
2216
  and ("access no." in t)
@@ -2222,21 +1867,6 @@ def _is_nfcu_document(pdf_path: str) -> bool:
2222
 
2223
 
2224
  def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2225
- """
2226
- Full text-layer parser for Navy Federal Credit Union statements.
2227
- GLM-OCR fails on NFCU pages because:
2228
- - The summary table has 5 wide columns with no visible borders
2229
- - Transaction tables have Amount($) and Balance($) spatially far right
2230
- - Multi-line descriptions (wrapped lines) confuse OCR table detection
2231
- This function reads the PDF text layer directly via pdfplumber and produces
2232
- correct, complete HTML for every section: summary table, transaction tables,
2233
- items paid, savings, disclosures.
2234
- Detection guards (all must pass — safe no-op for all other banks):
2235
- 1. Page text must contain 'Access No.' AND 'Statement of Account'
2236
- (unique to Navy Federal statement format)
2237
- 2. pdfplumber must return extractable text (not a scanned page)
2238
- Returns complete HTML string for the page, or "" if not NFCU / any error.
2239
- """
2240
  try:
2241
  import pymupdf as fitz
2242
 
@@ -2260,7 +1890,6 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2260
  if not is_nfcu:
2261
  return ""
2262
 
2263
- # Group words into reading lines by y-position.
2264
  buckets = []
2265
  for w in words:
2266
  if len(w) < 5:
@@ -2322,12 +1951,10 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2322
  i += 1
2323
  continue
2324
 
2325
- # Section/account boundaries
2326
  if acct_re.match(text) and i != start_idx:
2327
  break
2328
  if any(text.startswith(s) for s in stop_markers):
2329
  break
2330
- # Some boilerplate lines include prefixes/suffixes; match by containment too.
2331
  upper_text = text.upper()
2332
  if (
2333
  "REMITTANCE RECEIVED" in upper_text
@@ -2364,7 +1991,6 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2364
  amount = mvals[-2]
2365
  balance = mvals[-1]
2366
  elif len(mvals) == 1:
2367
- # Many NFCU lines can be split; single amount token is usually balance.
2368
  balance = mvals[-1]
2369
 
2370
  pending = {
@@ -2378,8 +2004,6 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2378
  mvals = [tok for tok in toks if money_re.match(tok)]
2379
  desc_tokens = [tok for tok in toks if not money_re.match(tok)]
2380
 
2381
- # Stop if we have already captured a balance and now hit
2382
- # known non-transaction boilerplate lines.
2383
  upper = text.upper()
2384
  if pending["balance"] and (
2385
  "REMITTANCE" in upper
@@ -2391,8 +2015,6 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2391
  ):
2392
  break
2393
 
2394
- # Beginning/Ending balance rows are single logical rows.
2395
- # Do not absorb unrelated continuation text.
2396
  if (
2397
  ("BEGINNING BALANCE" in pending["desc"].upper() or "ENDING BALANCE" in pending["desc"].upper())
2398
  and not mvals
@@ -2408,12 +2030,10 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2408
  pending["balance"] = mvals[-1]
2409
  elif len(mvals) == 1 and not pending["balance"]:
2410
  pending["balance"] = mvals[-1]
2411
- # else ignore stray line
2412
  i += 1
2413
 
2414
  if pending is not None:
2415
  rows.append(pending)
2416
- # keep rows that have at least date+desc, and preferably balance
2417
  cleaned = [r for r in rows if r["date"] and r["desc"] and (r["amount"] or r["balance"])]
2418
  return cleaned, i
2419
 
@@ -2444,7 +2064,6 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2444
  i = i2
2445
  continue
2446
 
2447
- # Keep section headers only (avoid noisy paragraph dump)
2448
  if text in ("Checking", "Savings"):
2449
  out.append(f"<h3>{text}</h3>")
2450
  elif text.startswith("Items Paid"):
@@ -2458,11 +2077,6 @@ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2458
  return ""
2459
 
2460
  def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2461
- """
2462
- Navy-only additive helper:
2463
- Rebuild "Summary of your deposit accounts" from PDF text-layer words.
2464
- Returns a clean HTML table or "" (safe no-op on any miss/error).
2465
- """
2466
  try:
2467
  import pymupdf as fitz
2468
 
@@ -2476,7 +2090,6 @@ def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2476
  if not words:
2477
  return ""
2478
 
2479
- # Group tokens into reading lines by y-position.
2480
  buckets = []
2481
  for w in words:
2482
  if len(w) < 5:
@@ -2519,8 +2132,6 @@ def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2519
  re.IGNORECASE,
2520
  )
2521
 
2522
- # NFCU summary table real columns from PDF:
2523
- # Account | Previous Balance | Deposits/Credits | Withdrawals/Debits | Ending Balance | YTD Dividends
2524
  rows = []
2525
  pending_account = ""
2526
  account_name_re = re.compile(
@@ -2538,7 +2149,6 @@ def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2538
  t0 = toks[0].lower()
2539
  tl = text.lower()
2540
 
2541
- # Ignore split header lines ("Previous ...", "Balance Credits ...")
2542
  if (
2543
  "previous" in tl
2544
  or "deposits" in tl
@@ -2548,7 +2158,6 @@ def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2548
  ) and not any(money_re.match(t) for t in toks):
2549
  continue
2550
 
2551
- # Account name can appear on its own line, followed by acct# + numeric columns.
2552
  if account_name_re.match(text.strip()) and not any(money_re.match(t) for t in toks):
2553
  pending_account = text.strip()
2554
  continue
@@ -2556,12 +2165,10 @@ def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2556
  mvals = [t.replace("$", "") for t in toks if money_re.match(t)]
2557
  acct_tokens = [t for t in toks if acct_no_re.match(t)]
2558
 
2559
- # Totals row: "Totals <five amounts>"
2560
  if t0 == "totals" and len(mvals) >= 5:
2561
  rows.append(["Totals", mvals[0], mvals[1], mvals[2], mvals[3], mvals[4]])
2562
  continue
2563
 
2564
- # Account numeric row: "<acctno> <five amounts>"
2565
  if acct_tokens and len(mvals) >= 5:
2566
  acct = acct_tokens[0]
2567
  acct_label = (pending_account + " - " + acct).strip(" -") if pending_account else acct
@@ -2585,25 +2192,18 @@ def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2585
  return ""
2586
 
2587
  def _replace_nfcu_summary_table(page_md: str, summary_table_html: str) -> str:
2588
- """
2589
- Replace the first table after 'Summary of your deposit accounts' with
2590
- a rebuilt NFCU summary table. Safe no-op if anchor/table is missing.
2591
- """
2592
  if not page_md or not summary_table_html:
2593
  return page_md
2594
  table_pattern = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
2595
  lower = page_md.lower()
2596
  anchor = lower.find("summary of your deposit accounts")
2597
 
2598
- # Primary path: replace first table after the summary heading.
2599
  if anchor >= 0:
2600
  for m in table_pattern.finditer(page_md):
2601
  if m.start() > anchor:
2602
  old_table = m.group(0)
2603
  return page_md.replace(old_table, summary_table_html, 1)
2604
 
2605
- # Fallback path: heading may be malformed/absent in OCR text.
2606
- # For NFCU summary, header should contain balance/deposit/withdrawal/ytd signals.
2607
  for m in table_pattern.finditer(page_md):
2608
  tbl = m.group(0)
2609
  p = TableGridParser()
@@ -2624,11 +2224,6 @@ def _replace_nfcu_summary_table(page_md: str, summary_table_html: str) -> str:
2624
  return page_md
2625
 
2626
  def _split_fused_date_transaction_header(grid):
2627
- """
2628
- Fix fused header "Date Transaction Detail" -> separate Date + Transaction Detail.
2629
- Strict guard: only when header has that exact fused phrase and also has
2630
- Amount/Balance columns, so unrelated tables are untouched.
2631
- """
2632
  if not grid or len(grid) < 2:
2633
  return grid
2634
  ncols = max(len(r) for r in grid if isinstance(r, list))
@@ -2679,13 +2274,6 @@ def _split_fused_date_transaction_header(grid):
2679
  return new_grid
2680
 
2681
  def _split_combined_summary_daily_balance_grid(grid):
2682
- """
2683
- Some OCR outputs fuse an account summary table and a Daily Balance block
2684
- into one HTML table. Split them into two tables when structure is clear.
2685
- Generic guards:
2686
- - A row containing "Daily Balance" exists
2687
- - There are rows after it with date-like + amount-like tokens
2688
- """
2689
  if not grid or len(grid) < 4:
2690
  return None
2691
 
@@ -2719,7 +2307,6 @@ def _split_combined_summary_daily_balance_grid(grid):
2719
  for d, a in zip(dates, amts):
2720
  daily_rows.append([d, a])
2721
 
2722
- # Need meaningful daily table to split safely.
2723
  if len(daily_rows) < 2:
2724
  return None
2725
 
@@ -2731,20 +2318,12 @@ def _split_combined_summary_daily_balance_grid(grid):
2731
  return [summary, daily]
2732
 
2733
  def _normalize_double_desc_amount_header_table(grid):
2734
- """
2735
- Fix 13: Collapse OCR tables with a duplicated column group:
2736
- Date | Description | Amount | Description | Amount
2737
- (or 4 cols if trailing Amount column was dropped). UCB-style summary tables
2738
- put labels like "9 Credit(s) This Period" in the Date column.
2739
- Output: Date | Description | Amount
2740
- """
2741
  if not grid or len(grid) < 2:
2742
  return grid
2743
 
2744
  def _hdr_tight(s):
2745
  return re.sub(r"\s+", "", str(s or "").strip().lower())
2746
 
2747
- # OCR often inserts spaces inside words; compare whitespace-stripped tokens.
2748
  expected5_t = ["date", "description", "amount", "description", "amount"]
2749
  expected4_t = ["date", "description", "amount", "description"]
2750
 
@@ -2794,7 +2373,6 @@ def _normalize_double_desc_amount_header_table(grid):
2794
  if not any(cells[:ncols]):
2795
  continue
2796
 
2797
- # A) Fused date + label in col0, amount in col1 (Beginning/Ending Balance, etc.)
2798
  mpre = date_prefix_re.match(c0)
2799
  if mpre and _is_money(c1):
2800
  date_s = mpre.group(1).strip()
@@ -2808,7 +2386,6 @@ def _normalize_double_desc_amount_header_table(grid):
2808
  out.append([date_s, desc, c1])
2809
  continue
2810
 
2811
- # B) Date-only col0; description may contain Beginning Balance + amount; amount col wrong ($0)
2812
  if date_only_re.match(c0.strip()):
2813
  date_s = c0.strip()
2814
  desc = c1
@@ -2828,25 +2405,16 @@ def _normalize_double_desc_amount_header_table(grid):
2828
  out.append([date_s, desc, amt])
2829
  continue
2830
 
2831
- # D) Summary / label rows: first column is NOT a date, second is money (Credits/Debits/Service)
2832
  if not date_prefix_re.match(c0) and not date_only_re.match(c0.strip()) and _is_money(c1):
2833
  out.append(["", c0, c1])
2834
  continue
2835
 
2836
- # C) Default: first Date|Description|Amount triple (ignore duplicate pair)
2837
  out.append([c0, c1, c2])
2838
 
2839
  return out if len(out) >= 2 else grid
2840
 
2841
 
2842
  def _normalize_ucb_three_col_beginning_balance_fusion(grid):
2843
- """
2844
- Fix 14: UCB sometimes OCRs the summary as 3 columns (not 5), with the real
2845
- beginning balance inside the description and $0.00 in the amount column:
2846
- 02/01/2025 | Beginning Balance $3,562.49 Average Ledger Balance | $0.00
2847
- Normalize to: Date | Beginning Balance | $3,562.49
2848
- Guard: standard Date|Description|Amount header only.
2849
- """
2850
  if not grid or len(grid) < 2:
2851
  return grid
2852
 
@@ -2854,7 +2422,6 @@ def _normalize_ucb_three_col_beginning_balance_fusion(grid):
2854
  return re.sub(r"\s+", "", str(s or "").strip().lower())
2855
 
2856
  h0 = [str(c or "").strip() for c in grid[0]]
2857
- # Do not use max row width — OCR sometimes adds an extra empty column on some rows only.
2858
  if len(h0) < 3:
2859
  return grid
2860
  if _tight(h0[0]) != "date" or _tight(h0[1]) != "description" or _tight(h0[2]) != "amount":
@@ -2894,12 +2461,6 @@ def _normalize_ucb_three_col_beginning_balance_fusion(grid):
2894
 
2895
 
2896
  def _fix_three_col_date_desc_amount_left_shift(grid):
2897
- """
2898
- Fix 15: OCR shifts transaction rows one column left:
2899
- col0 = "02/24/2025 ITM DEPOSIT", col1 = "$1,440.00", col2 = empty
2900
- Expected: Date | Description | Amount
2901
- Activates only on Date|Description|Amount headers and strict money in col1 with empty col2.
2902
- """
2903
  if not grid or len(grid) < 2:
2904
  return grid
2905
 
@@ -2948,14 +2509,6 @@ def _fix_three_col_date_desc_amount_left_shift(grid):
2948
 
2949
 
2950
  def _normalize_fused_date_posted_amount_table(grid):
2951
- """
2952
- Normalize 2-column OCR tables with fused header like:
2953
- col1: "Date posted Transaction description [Reference number]"
2954
- col2: "Amount"
2955
- into stable 3-column transaction layout:
2956
- Date | Transaction Description | Amount
2957
- Strict guard: only activates on that specific fused-header signature.
2958
- """
2959
  if not grid or len(grid) < 2:
2960
  return grid
2961
 
@@ -2988,7 +2541,6 @@ def _normalize_fused_date_posted_amount_table(grid):
2988
  out.append(["Date", "Transaction Description", "Amount"])
2989
  continue
2990
  if ri < hdr_idx:
2991
- # Keep pre-header text as non-transaction row.
2992
  joined = " ".join(c for c in cells if c).strip()
2993
  out.append(["", joined, ""])
2994
  continue
@@ -2996,7 +2548,6 @@ def _normalize_fused_date_posted_amount_table(grid):
2996
  c1 = cells[0].strip()
2997
  c2 = cells[1].strip() if len(cells) > 1 else ""
2998
 
2999
- # Skip duplicated inline sub-headers that often appear in fused OCR tables.
3000
  lower_c1 = c1.lower()
3001
  if (
3002
  lower_c1.startswith("date posted transaction description")
@@ -3012,8 +2563,6 @@ def _normalize_fused_date_posted_amount_table(grid):
3012
  date = parts[0]
3013
  desc = " ".join(parts[1:]).strip()
3014
 
3015
- # Amount may live in any shifted column for malformed OCR tables.
3016
- # Prefer explicit second-column value, then rightmost strict money token in row.
3017
  amount = c2
3018
  if not amount:
3019
  money_cells = []
@@ -3024,7 +2573,6 @@ def _normalize_fused_date_posted_amount_table(grid):
3024
  if money_cells:
3025
  amount = money_cells[-1]
3026
 
3027
- # If still empty, try extracting trailing amount from description.
3028
  if not amount and desc:
3029
  m = re.search(r"(-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?)\s*$", desc)
3030
  if m:
@@ -3035,23 +2583,11 @@ def _normalize_fused_date_posted_amount_table(grid):
3035
  continue
3036
  out.append([date, desc, amount])
3037
 
3038
- # Keep at least header + 1 row to avoid degrading clean tables on accidental match.
3039
  if len(out) < 2:
3040
  return grid
3041
  return out
3042
 
3043
  def normalize_html_tables(text: str) -> str:
3044
- """
3045
- For every <table>...</table>:
3046
- 1. Parse to grid (expand colspan/rowspan).
3047
- 2. Drop truly-empty columns.
3048
- 3. Merge blank-header text columns into the left column.
3049
- 4. Fix 1 — clean fused header artifacts and recover the lost first data row.
3050
- 5. Fix 2 — promote a misplaced header row when real column headers landed
3051
- in a <td> data row instead of the <th> header row.
3052
- 6. Fix 3 — split fused key-value rows in summary sections.
3053
- 7. Emit normalised <table> HTML.
3054
- """
3055
  if not text or "<table" not in text.lower():
3056
  return text
3057
 
@@ -3067,67 +2603,44 @@ def normalize_html_tables(text: str) -> str:
3067
  p.feed(table_html)
3068
  grid = _build_grid(p.rows)
3069
 
3070
- # Suppress known junk tables (OCR artifacts from page headers)
3071
  if _is_junk_table(grid):
3072
- # Append nothing (suppress the table), update last, skip rest of pipeline
3073
  out.append("")
3074
- last = m.end() # must set here before continue since line below is outside try
3075
  continue
3076
 
3077
- # Fix 13: MUST run before _drop_truly_empty_columns — otherwise the 5th column
3078
- # can be removed and the UCB double-pair header no longer matches.
3079
  grid = _normalize_double_desc_amount_header_table(grid)
3080
- # Fix 14: same bank, 3-col OCR fused Beginning Balance + $0.00 amount column.
3081
  grid = _normalize_ucb_three_col_beginning_balance_fusion(grid)
3082
- # Fix 15: "Deposits (continued)" etc. — date+desc in col1, amount in col2, empty col3.
3083
  grid = _fix_three_col_date_desc_amount_left_shift(grid)
3084
 
3085
  grid = _drop_truly_empty_columns(grid)
3086
  grid = _merge_blank_header_text_columns(grid)
3087
 
3088
- # Fix 5: reconstruct mid-table separator rows split across columns by OCR
3089
  grid = _reconstruct_separator_rows(grid)
3090
 
3091
- # Fix 1: fused-header artifact (two-signal guard — safe on clean PDFs)
3092
  grid, recovered_row = _clean_header_artifacts(grid)
3093
  if recovered_row is not None:
3094
  grid.insert(1, recovered_row)
3095
 
3096
- # Fix 2: misplaced header row promotion (keyword-score guard — safe on clean PDFs)
3097
  grid = _promote_misplaced_header_row(grid)
3098
 
3099
- # Fix NF-2: split fused "Date Transaction Detail" header/cell.
3100
  grid = _split_fused_date_transaction_header(grid)
3101
 
3102
- # Fix 11: normalize fused "Date posted Transaction description ... / Amount"
3103
- # two-column tables into stable Date/Description/Amount structure.
3104
  grid = _normalize_fused_date_posted_amount_table(grid)
3105
 
3106
- # Fix 8: split fused multi-keyword header cell (First Horizon Bank)
3107
  grid = _split_fused_multicolumn_header(grid)
3108
 
3109
- # Fix 9: normalize DAILY BALANCE SUMMARY tables with stacked dates
3110
  grid = _normalize_daily_balance_table(grid)
3111
 
3112
- # Fix 10: normalize CHECKS PAID SUMMARY repeating-group tables
3113
  grid = _normalize_checks_paid_table(grid)
3114
 
3115
- # Fix 6: merge split rows where description wraps to next line
3116
- # (Hardin County Bank and similar monospace statement formats)
3117
  grid = _merge_split_rows(grid)
3118
 
3119
- # Fix 7: extract trailing amount fused into description cell
3120
- # (Hardin County Bank: "CHASE CREDIT CRD EPAY 8319249882 500.00")
3121
  grid = _extract_fused_desc_amount(grid)
3122
 
3123
- # Fix 19: junk Amount cell but money token still present in row text (Truist MSBILL/PIN).
3124
  grid = _repair_da3_garbled_amount_cells(grid)
3125
 
3126
- # Fix 3: fused key-value rows in summary sections (no-space guard — safe on all PDFs)
3127
  grid = _fix_fused_keyvalue_rows(grid)
3128
- # Fix 16/20: duplicate data rows inside same Date|Description|Amount table (OCR repeats).
3129
  grid = _dedupe_duplicate_rows_in_da3_table(grid)
3130
- # Fix 12: split fused summary+daily-balance combined table, if detected.
3131
  split_tables = _split_combined_summary_daily_balance_grid(grid)
3132
  if split_tables:
3133
  out.append("\n\n".join(_grid_to_html(g) for g in split_tables if g))
@@ -3142,7 +2655,6 @@ def normalize_html_tables(text: str) -> str:
3142
 
3143
 
3144
  def _transaction_row_dedupe_key(cells):
3145
- """Stable key for Date|Description|Amount rows (OCR variants, HTML entities)."""
3146
  parts = [str(c or "").strip() for c in cells[:3]]
3147
  while len(parts) < 3:
3148
  parts.append("")
@@ -3150,16 +2662,12 @@ def _transaction_row_dedupe_key(cells):
3150
  date_s = re.sub(r"\s+", "", date_s)
3151
  amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
3152
  desc_s = html.unescape(desc_s)
3153
- desc_s = desc_s.replace("`", "'").replace("", "'")
3154
  desc_s = re.sub(r"\s+", "", desc_s.lower())
3155
  return (date_s, desc_s, amt_s)
3156
 
3157
 
3158
  def _da3_loose_key(cells):
3159
- """
3160
- (date, amount) only — for detecting duplicate DA3 tables when description drifts between OCR passes.
3161
- Uses strict currency + float amount so junk cells do not create keys; 1,234.56 vs 1234.56 match.
3162
- """
3163
  parts = [str(c or "").strip() for c in cells[:3]]
3164
  while len(parts) < 3:
3165
  parts.append("")
@@ -3181,7 +2689,6 @@ def _da3_loose_key(cells):
3181
 
3182
 
3183
  def _row_counter_loose_da3(g):
3184
- """Multiset of (date, amount) for DA3 data rows with strict amounts."""
3185
  c = Counter()
3186
  if not g or len(g) < 2:
3187
  return c
@@ -3196,7 +2703,6 @@ def _row_counter_loose_da3(g):
3196
 
3197
 
3198
  def _overlap_ratio_counter(c_num, c_den):
3199
- """Fraction of c_num's multiset mass matched in c_den (min counts per key)."""
3200
  tot = sum(c_num.values())
3201
  if tot == 0:
3202
  return 0.0
@@ -3228,11 +2734,6 @@ _DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
3228
 
3229
 
3230
  def _repair_da3_garbled_amount_cells(grid):
3231
- """
3232
- Fix 19: Amount column shows fused junk (letters, card digits) while the real debit/credit
3233
- still appears as a money token in the row text — use the rightmost plausible amount.
3234
- Only for Date|Description|Amount tables and strict MM/DD dates in column 0.
3235
- """
3236
  if not grid or len(grid) < 2 or not _is_da3_header(grid):
3237
  return grid
3238
 
@@ -3290,11 +2791,6 @@ def _repair_da3_garbled_amount_cells(grid):
3290
 
3291
 
3292
  def _dedupe_duplicate_rows_in_da3_table(grid):
3293
- """
3294
- Fix 16 + 20: Drop long runs of OCR-identical rows; allow up to 2 consecutive rows with the
3295
- same normalized Date|Description|Amount key (legitimate duplicate charges). Non-consecutive
3296
- repeats are kept (each run capped separately).
3297
- """
3298
  if not grid or len(grid) < 2:
3299
  return grid
3300
  if not _is_da3_header(grid):
@@ -3331,7 +2827,6 @@ def _dedupe_duplicate_rows_in_da3_table(grid):
3331
 
3332
 
3333
  def _row_text_full_ucb(grid, r: int) -> str:
3334
- """Lowercased join of all cells in row (for UCB section split heuristics)."""
3335
  if not grid or r < 0 or r >= len(grid):
3336
  return ""
3337
  parts = [str(c or "").strip().lower() for c in grid[r]]
@@ -3346,7 +2841,6 @@ def _is_itm_deposit_row_ucb(grid, r: int) -> bool:
3346
 
3347
 
3348
  def _ucb_verify_electronic_credits_rows(grid, ec_start: int) -> bool:
3349
- """POS Return, Acct Fund (Apple instant), DDA Transfer IN — PDF Electronic Credits block."""
3350
  if ec_start + 2 >= len(grid):
3351
  return False
3352
  t0 = _row_text_full_ucb(grid, ec_start)
@@ -3359,10 +2853,6 @@ def _ucb_verify_electronic_credits_rows(grid, ec_start: int) -> bool:
3359
 
3360
 
3361
  def _ucb_split_grid_deposits_ec_ed(grid):
3362
- """
3363
- Return (grid_dep, grid_ec, grid_ed) each with same Date|Description|Amount header,
3364
- or None if this does not look like the merged UCB page-3 pattern.
3365
- """
3366
  if not grid or len(grid) < 2:
3367
  return None
3368
  if not _is_da3_header(grid):
@@ -3390,10 +2880,6 @@ def _ucb_split_grid_deposits_ec_ed(grid):
3390
 
3391
 
3392
  def _try_split_ucb_deposits_table_to_three(prefix: str, table_html: str):
3393
- """
3394
- Build HTML: Deposits (continued) + table, Electronic Credits + table, Electronic Debits + table.
3395
- Returns None if split heuristics do not match.
3396
- """
3397
  try:
3398
  p = TableGridParser()
3399
  p.feed(table_html)
@@ -3421,11 +2907,6 @@ _UCB_DEP_CONT_BLOCK = re.compile(
3421
 
3422
 
3423
  def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
3424
- """
3425
- Fix 17: Split merged UCB Deposits (continued) table into Deposits + Electronic Credits + Electronic Debits.
3426
- Removes orphan empty Electronic Credits / Electronic Debits div pairs when replaced by real tables.
3427
- Safe no-op when heuristics do not match.
3428
- """
3429
  if not text or "deposits (continued)" not in text.lower():
3430
  return text
3431
 
@@ -3441,7 +2922,6 @@ def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
3441
 
3442
 
3443
  def _parse_da3_grid_from_table_html(table_html: str):
3444
- """Return grid if table is Date|Description|Amount; else None."""
3445
  try:
3446
  p = TableGridParser()
3447
  p.feed(table_html)
@@ -3454,11 +2934,6 @@ def _parse_da3_grid_from_table_html(table_html: str):
3454
 
3455
 
3456
  def _strip_orphan_continued_da3_flatline(text: str) -> str:
3457
- """
3458
- Fix 22: Remove one-line OCR dumps like
3459
- '…(continued) DATE DESCRIPTION AMOUNT($) 04/09 … 04/11 … 239.55' that duplicate the
3460
- following HTML table and pollute markdown.
3461
- """
3462
  if not text or "(continued)" not in text.lower():
3463
  return text
3464
  lines = text.splitlines(keepends=True)
@@ -3481,10 +2956,6 @@ def _strip_orphan_continued_da3_flatline(text: str) -> str:
3481
 
3482
 
3483
  def _strip_standalone_continued_banner_line(text: str) -> str:
3484
- """
3485
- Fix 27: Remove a single line that is only an account/section “(continued)” banner
3486
- (no transaction columns), often duplicated between page-separator and the next heading/table.
3487
- """
3488
  if not text or "(continued)" not in text.lower():
3489
  return text
3490
  lines = text.splitlines(keepends=True)
@@ -3512,10 +2983,6 @@ def _strip_standalone_continued_banner_line(text: str) -> str:
3512
 
3513
 
3514
  def _patch_da3_msft_g_ref_amounts_across_tables(text: str) -> str:
3515
- """
3516
- Fix 21: Build MICROSOFT#G… -> strict Amount from any DA3 row, then patch other DA3 rows
3517
- with the same id whose Amount cell is not strict currency (junk fused into Amount).
3518
- """
3519
  if not text or "<table" not in text.lower():
3520
  return text
3521
  if "microsoft#" not in text.lower():
@@ -3604,12 +3071,6 @@ def _patch_da3_msft_g_ref_amounts_across_tables(text: str) -> str:
3604
 
3605
 
3606
  def _trim_adjacent_da3_suffix_duplicating_next_table_prefix(text: str) -> str:
3607
- """
3608
- Remove trailing data rows from a DA3 table when they are repeated as the opening
3609
- rows of the immediately following DA3 table (OCR/LLM glued the next section onto
3610
- the prior table). Requires overlap length >= 2 and leaves at least one data row
3611
- in the first table. Safe no-op when patterns do not match.
3612
- """
3613
  if not text or "<table" not in text.lower():
3614
  return text
3615
 
@@ -3662,19 +3123,6 @@ def _trim_adjacent_da3_suffix_duplicating_next_table_prefix(text: str) -> str:
3662
 
3663
 
3664
  def _dedupe_subset_transaction_tables_html(text: str) -> str:
3665
- """
3666
- Remove duplicate 3-column Date|Description|Amount tables:
3667
- - Later table is a strict subset of an earlier one → drop later (original).
3668
- - Earlier table is a strict subset of a later one → drop earlier (UCB-style
3669
- "Deposits (continued)" preview before the same rows appear in the full list).
3670
- - Identical row sets → drop the later table.
3671
- - Fuzzy overlap (≥92% of later rows match earlier) → drop later (Electronic Debits vs main).
3672
- - Fix 23: multiset overlap on (date, amount) only — drop when OCR varies description
3673
- between two copies of the same section (e.g. withdrawals pasted twice).
3674
- - Runs multiple passes until stable.
3675
- Row matching uses normalized keys so OCR variants (e.g. "0 27" vs "027", backticks)
3676
- do not prevent duplicate tables from being detected.
3677
- """
3678
  if not text or "<table" not in text.lower():
3679
  return text
3680
 
@@ -3729,12 +3177,10 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
3729
  if not sj:
3730
  continue
3731
 
3732
- # Earlier j fully contained in later i (preview + full list) → drop earlier.
3733
  if sj <= si and len(sj) < len(si):
3734
  drop_idx.add(j)
3735
  continue
3736
 
3737
- # Later i strict subset of earlier j → drop later (e.g. Electronic Debits).
3738
  if si <= sj and len(si) < len(sj):
3739
  drop_idx.add(i)
3740
  break
@@ -3744,7 +3190,6 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
3744
  break
3745
 
3746
  inter = si & sj
3747
- # Later table's rows almost all appear in earlier table (OCR drift).
3748
  ri = len(inter) / len(si) if si else 0.0
3749
  rj = len(inter) / len(sj) if sj else 0.0
3750
  if ri >= 0.88 and len(si) <= len(sj):
@@ -3754,14 +3199,12 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
3754
  drop_idx.add(j)
3755
  continue
3756
 
3757
- # Fix 23: multiset (date, amount) overlap — duplicate tables when description OCR differs.
3758
  ci = _row_counter_loose_da3(gi)
3759
  cj = _row_counter_loose_da3(gj)
3760
  ni, nj = sum(ci.values()), sum(cj.values())
3761
  if ni >= 1 and nj >= 1:
3762
  oi_j = _overlap_ratio_counter(ci, cj)
3763
  oj_i = _overlap_ratio_counter(cj, ci)
3764
- # Short re-printed fragment (often 3–5 rows after a repeated header on same PDF page)
3765
  if (
3766
  nj >= 8
3767
  and 3 <= ni <= 14
@@ -3813,10 +3256,6 @@ _RE_CENTER_DIV_TWO_DA3_TABLES = re.compile(
3813
 
3814
 
3815
  def _da3_row_credit_vs_debit_category(desc: str) -> str:
3816
- """
3817
- Generic classification for Date|Description|Amount rows (middle column only).
3818
- Used to detect OCR pasting a withdrawals table after a deposits heading.
3819
- """
3820
  d = (desc or "").lower()
3821
  if "debit card return" in d:
3822
  return "credit"
@@ -3830,7 +3269,6 @@ def _da3_row_credit_vs_debit_category(desc: str) -> str:
3830
  return "credit"
3831
  if "lrm claims" in d or ("claims" in d and "adj" in d):
3832
  return "credit"
3833
- # Credits / tax refunds that omit the word "deposit" (common on statement continuations)
3834
  if "adp tax" in d and "customer" in d:
3835
  return "credit"
3836
  if "debit card" in d:
@@ -3853,7 +3291,6 @@ def _da3_row_credit_vs_debit_category(desc: str) -> str:
3853
 
3854
 
3855
  def _da3_table_credit_debit_fractions(grid):
3856
- """Returns (debit_frac, credit_frac, unknown_frac, scored_row_count)."""
3857
  deb = cre = unk = 0
3858
  if not grid or len(grid) < 2:
3859
  return 0.0, 0.0, 0.0, 0
@@ -3879,12 +3316,6 @@ def _da3_table_credit_debit_fractions(grid):
3879
 
3880
 
3881
  def _drop_da3_misplaced_debit_after_deposit_heading_tables(text: str) -> str:
3882
- """
3883
- Fix 25: PDF order is withdrawals → then deposits. OCR sometimes emits two DA3 tables
3884
- after the deposits heading: real deposits, then a duplicate withdrawals block.
3885
- Drop the second table when it is overwhelmingly debit-like and the first is credit-like.
3886
- Repeats until no more matches (multiple duplicate blocks).
3887
- """
3888
  if not text or "<table" not in text.lower():
3889
  return text
3890
  if "deposit" not in text.lower():
@@ -3916,7 +3347,6 @@ def _drop_da3_misplaced_debit_after_deposit_heading_tables(text: str) -> str:
3916
  if n1 < 5 or n2 < 10:
3917
  pos = m.start() + 1
3918
  continue
3919
- # First table: deposits often have ADP/merchant lines scored as unknown — blend unk toward credit.
3920
  credit_like_first = cr1 + 0.42 * unk1
3921
  if credit_like_first < 0.30 and credit_like_first <= db1 + 0.05:
3922
  pos = m.start() + 1
@@ -3934,6 +3364,152 @@ def _drop_da3_misplaced_debit_after_deposit_heading_tables(text: str) -> str:
3934
  return text
3935
 
3936
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3937
  def stabilize_tables_and_text(page_md: str) -> str:
3938
  if not page_md:
3939
  return page_md
@@ -3958,6 +3534,7 @@ def stabilize_tables_and_text(page_md: str) -> str:
3958
  stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
3959
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
3960
  stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
 
3961
  stabilized = _strip_orphan_continued_da3_flatline(stabilized)
3962
  stabilized = _strip_standalone_continued_banner_line(stabilized)
3963
  return close_unclosed_html(stabilized)
@@ -4062,10 +3639,6 @@ def run_ocr(uploaded_file):
4062
 
4063
  parts = []
4064
 
4065
- # Keep GLM-OCR as the primary parser for all pages.
4066
- # Navy-specific helpers run only as additive post-processing.
4067
-
4068
- # Header inclusion: PDF text -> band words -> OCR band
4069
  hdr = ""
4070
  if is_pdf:
4071
  hdr = extract_zone_text_pdf(path, page_num, 0, he)
@@ -4076,39 +3649,30 @@ def run_ocr(uploaded_file):
4076
  if hdr and hdr.strip():
4077
  parts.append(normalize_html_tables(fix_account_number(normalize_money_glyphs(hdr.strip()))))
4078
 
4079
- # Main OCR body: stabilize then patch missing duplicate rows from text layer
4080
  if page_md and page_md.strip():
4081
  stabilized = stabilize_tables_and_text(page_md.strip())
4082
- # Fix 4: restore any rows OCR dropped by cross-checking the PDF text layer.
4083
- # Safe no-op for scanned PDFs (no text layer) and non-transaction pages.
4084
  if is_pdf and not is_nfcu_doc:
4085
  stabilized = _patch_ocr_with_textlayer(stabilized, path, page_num)
4086
  stabilized = _strip_orphan_continued_da3_flatline(stabilized)
4087
  stabilized = _strip_standalone_continued_banner_line(stabilized)
4088
- # _patch_ocr_with_textlayer may append Case B rows from the PDF text layer
4089
- # with fused UCB-style cells; run the same HTML table pass again (Fix 13/14).
4090
  stabilized = normalize_html_tables(stabilized)
4091
  stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
4092
  stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
4093
  stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
4094
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
4095
  stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
 
4096
  stabilized = _strip_standalone_continued_banner_line(stabilized)
4097
  elif is_pdf and is_nfcu_doc:
4098
- # Navy-only additive fix: rebuild malformed summary table from
4099
- # text layer, but keep GLM body/header/footer unchanged.
4100
  nfcu_summary = _extract_nfcu_summary_table(path, page_num)
4101
  if nfcu_summary:
4102
  stabilized = _replace_nfcu_summary_table(stabilized, nfcu_summary)
4103
 
4104
- # Fix 4D: append Johnson Bank Checks + Daily Account Balance
4105
- # Run OUTSIDE _patch_ocr_with_textlayer so exceptions there don't block it.
4106
  if is_pdf:
4107
  try:
4108
  needs_checks = 'class="jb-summary-checks"' not in stabilized
4109
  needs_dab = "daily account balance" not in stabilized.lower()
4110
  if needs_checks or needs_dab:
4111
- # Try pdfminer first (column-aware), fall back to pymupdf
4112
  _txt4d = ""
4113
  try:
4114
  from pdfminer.high_level import extract_text as _pm_extract
@@ -4128,12 +3692,10 @@ def run_ocr(uploaded_file):
4128
  if jb_extra:
4129
  stabilized = stabilized.rstrip() + "\n\n" + jb_extra
4130
  except Exception:
4131
- pass # safe no-op — never break other PDFs
4132
 
4133
  parts.append(stabilized)
4134
 
4135
- # Footer extraction: PDF text layer -> band words -> OCR crop
4136
- # Deduplication guard prevents double-printing when body OCR already captured it.
4137
  if ENABLE_FOOTER_OCR and page_num < len(page_images):
4138
  ftr = ""
4139
  if is_pdf:
@@ -4145,7 +3707,6 @@ def run_ocr(uploaded_file):
4145
  if ftr and ftr.strip():
4146
  ftr_clean = normalize_money_glyphs(ftr.strip())
4147
 
4148
- # Guard 1: first-line dedup (original check)
4149
  ftr_first_line = next(
4150
  (ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
4151
  ""
@@ -4154,11 +3715,6 @@ def run_ocr(uploaded_file):
4154
  ftr_first_line in part.lower() for part in parts
4155
  )
4156
 
4157
- # Guard 2: suppress footer that is a raw text-layer dump of
4158
- # transaction rows (e.g. East West Bank two-column layout).
4159
- # Detected when the footer contains 3+ date tokens (MM/DD or MM-DD)
4160
- # AND the footer itself contains amounts — meaning it is transaction
4161
- # data, not a legitimate page footer like an address or disclaimer.
4162
  _footer_date_re = re.compile(r"\b\d{1,2}[-/]\d{2}\b")
4163
  _footer_amt_re = re.compile(r"\b\d{1,3}(?:,\d{3})*\.\d{2}\b")
4164
  _date_hits = len(_footer_date_re.findall(ftr_clean))
@@ -4172,7 +3728,6 @@ def run_ocr(uploaded_file):
4172
  all_pages.append("\n\n".join(parts))
4173
 
4174
  merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
4175
- # Cross-page: UCB section split + same transaction tables dedupe once on full doc.
4176
  if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
4177
  merged = _strip_orphan_continued_da3_flatline(merged)
4178
  merged = _strip_standalone_continued_banner_line(merged)
@@ -4181,6 +3736,7 @@ def run_ocr(uploaded_file):
4181
  merged = _patch_da3_msft_g_ref_amounts_across_tables(merged)
4182
  merged = _dedupe_subset_transaction_tables_html(merged)
4183
  merged = _drop_da3_misplaced_debit_after_deposit_heading_tables(merged)
 
4184
  merged = _strip_orphan_continued_da3_flatline(merged)
4185
  merged = _strip_standalone_continued_banner_line(merged)
4186
  return merged
 
41
  - Fix 21: Cross-table DA3 repair for MSBILL / MICROSOFT#G… PIN rows — when Amount is junk in one table
42
  but another table repeats the same authorization id with a strict amount, copy that amount
43
  (then subset dedupe can drop the stray duplicate block, e.g. under Deposits).
44
+ - Fix 22: Drop a single OCR "flat line" between pages: (continued) + DESCRIPTION + AMOUNT + many MM/DD
45
  tokens (noise before a proper <table>).
46
  - Fix 23: Doc-wide DA3 table dedupe using multiset of (date, amount) keys — catches duplicate sections when
47
  OCR varies description text between copies (e.g. withdrawals repeated under Deposits).
48
  - Fix 24: Run orphan flatline strip on merged PDF output and after text-layer patch so page-boundary
49
  noise is removed consistently.
50
+ - Fix 25: After a centered "deposits + credits/interest" heading, if two DA3 tables appear back-to-back
51
  and the first is mostly credit/deposit rows while the second is mostly debit/withdrawal-style
52
  rows (generic keywords), drop the second — fixes OCR pasting the withdrawals block again.
53
  - Fix 23b: Loose multiset keys use normalized float amounts so comma/spacing variants still match.
54
  - Fix 26: Drop small DA3 fragments (3–14 rows) that are a multiset subset of an earlier larger DA3
55
  table — matches PDF layouts that re-print a header and repeat a few rows.
56
+ - Fix 27: Strip a lone "…(continued)" account banner line (no DATE/DESCRIPTION/AMOUNT) between
57
  page separators — duplicate header noise without the long flatline.
58
+ - Fix 28: Strip leading credit/deposit rows that OCR prepends to a withdrawals/debits DA3 table
59
+ when the preceding DA3 table was under a deposits/credits heading. Only strips the
60
+ contiguous prefix of the debit table whose rows appear in the credit table; requires
61
+ ≥1 data row to remain; entire function wrapped in try/except (safe no-op on any error).
62
  """
63
 
64
  # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
 
616
  # --------------------------
617
 
618
  def _split_keyword_remainder(cell_text: str):
 
 
 
 
619
  s = cell_text.strip()
620
  for pattern in _HEADER_KW_RES:
621
  m = pattern.match(s)
 
626
  return s, ""
627
 
628
  def _extract_fused_header_artifacts(header_row):
 
 
 
 
 
 
 
 
 
 
 
 
629
  if not header_row:
630
  return list(header_row), None
631
 
 
664
  return list(header_row), None
665
 
666
  def _clean_header_artifacts(grid):
 
 
 
 
667
  if not grid or not grid[0]:
668
  return grid, None
669
 
 
676
  # --------------------------
677
 
678
  def _row_keyword_score(row):
 
 
 
 
 
 
679
  non_empty = [str(c or "").strip() for c in row if str(c or "").strip()]
680
  if not non_empty:
681
  return 0.0
 
692
 
693
  return keyword_hits / len(non_empty)
694
 
 
 
695
  _BALANCE_INFO_RE = re.compile(
696
  r"(beginning|ending|opening|closing)\s+balance"
697
  r"|account\s*#?\s*\d{5,}"
 
701
  )
702
 
703
  def _header_contains_balance_info(header_row):
 
 
 
 
 
704
  for cell in header_row:
705
  if _BALANCE_INFO_RE.search(str(cell or "")):
706
  return True
707
  return False
708
 
709
  def _promote_misplaced_header_row(grid):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
710
  if not grid or len(grid) < 2:
711
  return grid
712
 
 
714
  if current_header_score >= _HEADER_ROW_KEYWORD_THRESHOLD:
715
  return grid
716
 
 
717
  candidate_idx = None
718
  candidate_score = 0.0
719
  for i in range(1, min(len(grid), 6)):
 
727
 
728
  candidate_row = grid[candidate_idx]
729
 
 
730
  expanded_header = []
731
  for cell in candidate_row:
732
  cell_s = str(cell or "").strip()
 
739
  new_ncols = len(expanded_header)
740
  col_expansion = new_ncols - len(candidate_row)
741
 
 
 
 
742
  metadata_row = None
743
  if _header_contains_balance_info(grid[0]):
744
  meta_cells = [str(c or "").strip() for c in grid[0]]
 
746
  if meta_text:
747
  metadata_row = [meta_text] + [""] * (new_ncols - 1)
748
 
 
749
  new_data_rows = []
750
  for row in grid[candidate_idx + 1:]:
751
  if col_expansion > 0 and row:
 
766
  return r[:n]
767
 
768
  new_grid = [pad(expanded_header, new_ncols)]
 
769
  if metadata_row is not None:
770
  new_grid.append(pad(metadata_row, new_ncols))
771
  for row in new_data_rows:
 
777
  # Fix 3: Fused key-value rows in summary sections
778
  # --------------------------
779
 
 
 
 
 
780
  _FUSED_KV_RE = re.compile(
781
  r"^(.+?)(-?\d{1,3}(?:,\d{3})*(?:\.\d+)?%?)$",
782
  re.DOTALL,
783
  )
784
 
785
  def _fix_fused_keyvalue_rows(grid):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
786
  if not grid or len(grid) < 2:
787
  return grid
788
 
 
 
789
  _DOLLAR_AMOUNT_RE = re.compile(r"\$\s*\d{1,3}(?:,\d{3})*\.\d{2}")
 
790
  _LONG_NUMBER_RE = re.compile(r"\d{7,}")
791
 
792
  ncols = len(grid[0])
 
805
  new_grid.append(row)
806
  continue
807
 
 
 
808
  if _DOLLAR_AMOUNT_RE.search(first):
809
  new_grid.append(row)
810
  continue
811
 
 
 
812
  if _LONG_NUMBER_RE.search(first):
813
  new_grid.append(row)
814
  continue
815
 
 
 
 
816
  if re.search(r"#\s*[\dX]", first):
817
  new_grid.append(row)
818
  continue
 
822
  new_grid.append(row)
823
  continue
824
 
825
+ label_raw = m.group(1)
826
  value = m.group(2).strip()
827
 
 
828
  if label_raw != label_raw.rstrip():
829
  new_grid.append(row)
830
  continue
 
847
  # --------------------------
848
 
849
  def _extract_textlayer_rows(pdf_path: str, page_num: int):
 
 
 
 
 
 
 
 
 
 
850
  try:
851
  import pymupdf as fitz
852
 
 
858
  if not words:
859
  return []
860
 
 
861
  lines_by_y = {}
862
  for w in words:
863
  x0, y0, word = float(w[0]), float(w[1]), w[4]
 
875
  line_words = sorted(lines_by_y[y], key=lambda t: t[0])
876
  sorted_lines.append([w for _, w in line_words])
877
 
 
878
  _MONTHS = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
879
  date_re = re.compile(
880
+ r"^\d{1,2}/\d{2}(?:/\d{2,4})?$"
881
+ r"|^\d{1,2}-\d{2}(?:-\d{2,4})?$"
882
+ r"|^\d{4}-\d{2}-\d{2}$"
883
+ r"|^" + _MONTHS + r"$",
884
  re.IGNORECASE,
885
  )
 
886
  month_re = re.compile(r"^" + _MONTHS + r"$", re.IGNORECASE)
887
  day_re = re.compile(r"^\d{1,2}$")
888
 
 
889
  amount_re = re.compile(
890
  r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
891
  r"|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
892
+ r"|^-\s+\$\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
893
  )
894
 
895
  rows = []
 
897
  if len(line) < 2:
898
  continue
899
 
 
900
  date_str = None
901
  desc_start = None
902
 
903
  if date_re.match(line[0]):
904
  if month_re.match(line[0]) and len(line) > 1 and day_re.match(line[1]):
 
905
  date_str = line[0] + " " + line[1]
906
  desc_start = 2
907
  else:
 
908
  date_str = line[0]
909
  desc_start = 1
910
  else:
 
913
  if desc_start is None or desc_start >= len(line):
914
  continue
915
 
 
 
 
916
  remaining = line[desc_start:]
 
917
  post_date_str = ""
918
  if remaining and month_re.match(remaining[0]):
919
  if len(remaining) > 1 and day_re.match(remaining[1]):
 
928
  if len(remaining) < 2:
929
  continue
930
 
 
931
  if (len(remaining) >= 2
932
  and remaining[-2] == "-"
933
  and remaining[-1].startswith("$")):
 
944
  if not desc:
945
  continue
946
 
 
 
 
 
947
  if not re.search(r"[A-Za-z]", desc):
948
  continue
949
 
950
  rows.append({"date": date_str, "post_date": post_date_str, "desc": desc, "amount": amount_candidate})
951
 
 
952
  return rows if len(rows) >= 2 else []
953
 
954
  except Exception as e:
 
956
  return []
957
 
958
  def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
 
 
 
 
 
959
  if not text_layer:
960
  return []
961
  m = re.search(
 
984
  return out
985
 
986
  def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
 
 
 
 
 
 
 
 
 
 
987
  if not text_layer:
988
  return ""
989
 
 
994
  lines = [l.strip() for l in text_layer.splitlines()]
995
  output = []
996
 
 
997
  if needs_checks:
998
  checks = []
999
  i = 0
 
1051
  + rows + "\n</table>"
1052
  )
1053
 
 
1054
  if needs_dab:
1055
  balances = []
1056
  i = 0
 
1095
  return "\n\n".join(output)
1096
 
1097
  def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str:
 
 
 
 
 
 
 
 
 
 
 
 
1098
  if not page_md or "<table" not in page_md.lower():
1099
  return page_md
1100
 
1101
  try:
1102
  tl_rows = _extract_textlayer_rows(pdf_path, page_num)
1103
  if not tl_rows:
1104
+ return page_md
1105
 
1106
  def _norm(s):
1107
  return re.sub(r"\s+", " ", str(s or "").strip().lower())
1108
 
 
1109
  tl_freq = {}
1110
  for r in tl_rows:
1111
  key = (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"]))
 
1123
  if len(grid) < 2:
1124
  continue
1125
 
 
1126
  header = [str(c or "").strip().upper() for c in grid[0]]
1127
  date_col = desc_col = amt_col = None
1128
  for ci, h in enumerate(header):
 
1134
  amt_col = ci
1135
 
1136
  if date_col is None or desc_col is None or amt_col is None:
1137
+ continue
1138
 
 
1139
  ocr_freq = {}
1140
  ocr_rows_indexed = []
1141
  for ri, row in enumerate(grid[1:], start=1):
 
1151
  ocr_freq[key] = ocr_freq.get(key, 0) + 1
1152
  ocr_rows_indexed.append((ri, key))
1153
 
 
 
 
 
 
 
 
1154
  overlap = set(ocr_freq.keys()) & set(tl_freq.keys())
1155
 
 
1156
  _ocr_dates = [
1157
  _norm(str(row[date_col] or ""))
1158
  for row in grid[1:]
 
1169
  tl_fmt = _date_fmt(_tl_dates)
1170
  date_fmt_match = (ocr_fmt == tl_fmt) or "other" in (ocr_fmt, tl_fmt)
1171
 
 
 
 
1172
  to_inject = {}
1173
  to_inject_new = {}
1174
 
 
1178
  if tl_count > ocr_count:
1179
  missing = tl_count - ocr_count
1180
  if ocr_count > 0:
 
1181
  if overlap:
1182
  to_inject[key] = missing
1183
  else:
 
 
1184
  if date_fmt_match:
1185
  tl_row_data = next(
1186
  (r for r in tl_rows
 
1193
  if not to_inject and not to_inject_new:
1194
  continue
1195
 
 
1196
  new_grid = [grid[0]]
1197
  for ri, row in enumerate(grid[1:], start=1):
1198
  new_grid.append(row)
 
1213
  new_table_html = _grid_to_html(new_grid)
1214
  result = result.replace(table_html, new_table_html, 1)
1215
 
 
 
 
 
1216
  if to_inject_new:
1217
  ncols = len(grid[0])
1218
  header_row = grid[0]
1219
 
 
1220
  post_date_col = None
1221
  for ci, h in enumerate(header_row):
1222
  hh = str(h or "").strip().upper()
 
1242
  if new_rows:
1243
  extra_grid = [header_row] + new_rows
1244
  extra_html = "\n\n" + _grid_to_html(extra_grid)
 
1245
  insert_pos = result.find(new_table_html) + len(new_table_html)
1246
  result = result[:insert_pos] + extra_html + result[insert_pos:]
1247
 
 
 
 
 
1248
  if tl_rows:
1249
  _has_txn_table = False
1250
  for _tbl in table_pattern.finditer(result):
 
1284
 
1285
  except Exception as e:
1286
  log.warning("_patch_ocr_with_textlayer failed (page %d): %s", page_num, e)
1287
+ return page_md
1288
 
1289
  # --------------------------
1290
  # Fix 5: Mid-table separator row reconstruction
1291
  # --------------------------
1292
 
 
1293
  _DATE_RE = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
 
 
1294
  _SPLIT_AMOUNT_RE = re.compile(
1295
  r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
1296
  )
1297
 
1298
  def _reconstruct_separator_rows(grid):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1299
  if not grid or len(grid) < 2:
1300
  return grid
1301
 
 
1303
  if ncols < 2:
1304
  return grid
1305
 
1306
+ new_grid = [grid[0]]
1307
 
1308
  for row in grid[1:]:
1309
  cells = [str(c or "").strip() for c in row]
 
1312
 
1313
  non_empty = [(i, c) for i, c in enumerate(cells) if c]
1314
 
 
1315
  if len(non_empty) == 0:
1316
  new_grid.append(row)
1317
  continue
1318
 
1319
  first_idx, first_val = non_empty[0]
1320
 
 
1321
  if _DATE_RE.match(first_val):
1322
  new_grid.append(row)
1323
  continue
1324
 
 
1325
  if not re.search(r"[A-Za-z]", first_val):
1326
  new_grid.append(row)
1327
  continue
1328
 
 
1329
  if len(non_empty) == 1:
1330
  new_row = [""] * ncols
1331
  new_row[0] = first_val
1332
  new_grid.append(new_row)
1333
  continue
1334
 
 
 
 
 
 
 
1335
  trailing = [val for _, val in non_empty[1:]]
1336
  all_trailing_are_digit_fragments = all(
1337
  re.fullmatch(r"\d{1,4}", v) for v in trailing
 
1341
  new_grid.append(row)
1342
  continue
1343
 
 
1344
  reconstructed = "".join(val for _, val in non_empty).strip()
1345
  new_row = [""] * ncols
1346
  new_row[0] = reconstructed
 
1353
  # --------------------------
1354
 
1355
  def _merge_split_rows(grid):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1356
  if not grid or len(grid) < 3:
1357
  return grid
1358
 
 
1360
  if ncols < 3:
1361
  return grid
1362
 
 
1363
  header = [str(c or "").strip().upper() for c in grid[0]]
1364
  date_col = desc_col = bal_col = None
1365
  debit_cols = []
 
1377
  if re.search(r"\b(CREDIT|CREDITS|CR|DEPOSIT|ADDITIONS?)\b", h):
1378
  credit_cols.append(ci)
1379
 
 
1380
  if date_col is None or desc_col is None:
1381
  return grid
1382
  if not debit_cols and not credit_cols:
 
1385
  money_cols = debit_cols + credit_cols
1386
  _MONEY_RE2 = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*\.\d{1,4}$")
1387
  _DATE_RE2 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$|^\d{1,2}-\d{2}(?:-\d{2,4})?$")
 
1388
  _FRAGMENT_RE = re.compile(r"^[^\s\.]{1,6}$")
1389
 
1390
  new_grid = [grid[0]]
 
1394
  while len(row) < ncols:
1395
  row.append("")
1396
 
 
1397
  has_desc = bool(row[desc_col])
1398
  no_date = not row[date_col]
1399
  no_money = all(not row[c] for c in money_cols)
 
1409
  next_has_money = any(bool(next_row[c]) for c in money_cols)
1410
 
1411
  if next_has_date and next_has_money:
 
1412
  merged = list(next_row)
 
1413
  if next_row[desc_col]:
 
1414
  merged[desc_col] = row[desc_col] + " " + next_row[desc_col]
1415
  else:
 
1416
  merged[desc_col] = row[desc_col]
1417
  new_grid.append(merged)
1418
+ i += 2
1419
  continue
1420
 
1421
  new_grid.append(row)
 
1424
  return new_grid
1425
 
1426
  def _extract_fused_desc_amount(grid):
 
 
 
 
 
 
 
 
 
 
 
1427
  if not grid or len(grid) < 2:
1428
  return grid
1429
 
 
1474
  return new_grid
1475
 
1476
  def _is_junk_table(grid):
 
 
 
 
 
 
 
 
 
1477
  if not grid or len(grid) < 2:
1478
  return False
1479
  ncols = len(grid[0])
 
1481
  return False
1482
  header_text = str(grid[0][0] or "").strip().upper()
1483
  if "CUSTOMER INFORMATION" in header_text:
 
1484
  data_rows = grid[1:]
1485
  if all(len(str(r[0] or "").strip()) <= 10 for r in data_rows):
1486
  return True
1487
  return False
1488
 
1489
  def _split_fused_multicolumn_header(grid):
 
 
 
 
 
 
 
 
 
 
 
1490
  if not grid or len(grid) < 2:
1491
  return grid
1492
 
 
1495
  return grid
1496
 
1497
  first_header = str(grid[0][0] or "").strip()
 
1498
  parts = first_header.split()
1499
  kw_hits = [p for p in parts if _HEADER_KW_EXACT_RE.match(p)]
1500
  if len(kw_hits) < 2:
1501
+ return grid
1502
 
 
1503
  rest_headers = [str(c or "").strip() for c in grid[0][1:]]
1504
  if not all(_HEADER_KW_EXACT_RE.match(h) for h in rest_headers if h):
1505
  return grid
1506
 
 
 
 
1507
  new_header_cols = []
1508
  non_kw_parts = []
1509
  for p in parts:
 
1517
  if non_kw_parts:
1518
  new_header_cols.append(" ".join(non_kw_parts))
1519
 
 
1520
  new_header = new_header_cols + rest_headers
1521
  new_ncols = len(new_header)
1522
+ extra_cols = new_ncols - ncols
1523
 
 
1524
  _DATE_COL_RE = re.compile(r"\bDATE\b", re.IGNORECASE)
1525
  _CARD_COL_RE = re.compile(r"\bCARD\b", re.IGNORECASE)
1526
  _DESC_COL_RE = re.compile(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?)\b", re.IGNORECASE)
 
1531
  new_card_col = next((i for i,h in enumerate(new_header) if _CARD_COL_RE.search(h)), None)
1532
 
1533
  if new_date_col is None or new_desc_col is None:
1534
+ return grid
1535
 
 
1536
  _DATE_PREFIX = re.compile(r"^(\d{1,2}/\d{2}(?:/\d{2,4})?)\s+(.*)", re.DOTALL)
1537
+ _CARD_SUFFIX = re.compile(r"^(.*?)\s+(\d{4})$", re.DOTALL)
1538
 
1539
  def pad(row, n):
1540
  r = list(row)
 
1549
  fused_cell = cells[0]
1550
  rest_cells = cells[1:]
1551
 
 
1552
  date_val = ""
1553
  desc_val = fused_cell
1554
  card_val = ""
 
1558
  date_val = dm.group(1)
1559
  desc_val = dm.group(2).strip()
1560
 
 
1561
  if new_card_col is not None:
1562
  cm = _CARD_SUFFIX.match(desc_val)
1563
  if cm:
1564
  desc_val = cm.group(1).strip()
1565
  card_val = cm.group(2)
1566
 
 
1567
  new_row = [""] * new_ncols
1568
  new_row[new_date_col] = date_val
1569
  new_row[new_desc_col] = desc_val
1570
  if new_card_col is not None:
1571
  new_row[new_card_col] = card_val
 
1572
  money_col_indices = [i for i,h in enumerate(new_header) if _MONEY_COL_RE.search(h)]
1573
  for mi, ri in enumerate(range(len(rest_cells))):
1574
  if mi < len(money_col_indices):
 
1579
  return new_grid
1580
 
1581
  def _normalize_daily_balance_table(grid):
 
 
 
 
 
 
 
 
 
 
 
1582
  if not grid or len(grid) < 2:
1583
  return grid
1584
 
 
1597
  if not all(h == "BALANCE" for h in bal_half):
1598
  return grid
1599
 
 
1600
  new_header = []
1601
  for i in range(half):
1602
  new_header.append("DATE")
1603
  new_header.append("BALANCE")
1604
 
 
1605
  new_rows = []
1606
  _DATE_RE3 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
1607
 
 
1610
  while len(cells) < ncols:
1611
  cells.append("")
1612
 
 
 
 
1613
  date_cells = [cells[0]]
1614
  bal_cells = cells[1:]
1615
 
 
 
1616
  first = date_cells[0]
1617
  tokens = re.split(r"[\n\r\s]+", first)
1618
  date_tokens = [t.strip() for t in tokens if t.strip() and _DATE_RE3.match(t.strip())]
1619
  if len(date_tokens) >= 2:
1620
  stacked = date_tokens
1621
  else:
1622
+ stacked = date_cells
1623
 
 
1624
  new_row = [""] * ncols
1625
  for i, date_val in enumerate(stacked):
1626
  if not _DATE_RE3.match(date_val):
 
1636
  return [new_header] + new_rows
1637
 
1638
  def _normalize_checks_paid_table(grid):
 
 
 
 
 
 
 
 
 
 
1639
  if not grid or len(grid) < 2:
1640
  return grid
1641
 
 
1648
  if not _CHECKS_HDR.match(first_h):
1649
  return grid
1650
 
 
1651
  rest_h = [str(c or "").strip().upper() for c in grid[0][1:]]
1652
  if not all(h in ("AMOUNT", "") for h in rest_h if h):
1653
  return grid
1654
 
 
1655
  n_groups = len(re.findall(r"\bDATE\b", first_h, re.IGNORECASE))
1656
  if n_groups < 1:
1657
  return grid
1658
 
 
1659
  new_header = []
1660
  for _ in range(n_groups):
1661
  new_header.extend(["DATE", "CHECK #", "AMOUNT"])
 
1671
  fused_data = cells[0]
1672
  amounts = cells[1:]
1673
 
 
1674
  tokens = fused_data.split()
1675
  groups = []
1676
  current = []
 
1683
  if current and current[0]:
1684
  groups.append(current)
1685
 
 
1686
  new_row = [""] * len(new_header)
1687
  for i, grp in enumerate(groups):
1688
  if i >= n_groups:
1689
  break
1690
+ new_row[i * 3] = grp[0]
1691
+ new_row[i * 3 + 1] = " ".join(grp[1:]).strip()
1692
+ new_row[i * 3 + 2] = amounts[i] if i < len(amounts) else ""
1693
 
1694
  new_rows.append(new_row)
1695
 
1696
  return new_rows
1697
 
 
 
 
 
 
 
 
 
 
 
 
 
1698
  _JB_SECTION_HDR = re.compile(
1699
  r"^(Deposits?|Withdrawals?|Checks?(?:\s+Paid)?|Daily\s+Account\s+Balance)"
1700
  r"\s*(?:\(cont\.?\))?\s*$",
 
1705
  r"|^Date\s+(?:Number\s+)?(?:Balance|Amount)\s*$",
1706
  re.IGNORECASE,
1707
  )
 
1708
  _JB_DATE_DD = re.compile(r"^\d{2}-\d{2}\s+")
1709
  _JB_TXN_ROW = re.compile(r"^(\d{2}-\d{2})\s+(.+?)\s+(-?\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
1710
  _JB_BAL_ROW = re.compile(r"^(\d{2}-\d{2})\s+(\d{1,3}(?:,\d{3})*\.\d{2})\s*$")
 
1719
  return "\n".join(parts)
1720
 
1721
  def _jb_section_has_table(lines, start, end):
 
1722
  return any("<table" in l.lower() for l in lines[start:end])
1723
 
1724
  def _find_first_data_row(lines, start):
 
1725
  for i in range(start, min(start + 20, len(lines))):
1726
  if lines[i].strip():
1727
  return i
1728
  return None
1729
 
1730
  def convert_plaintext_bank_sections(text: str) -> str:
 
 
 
 
 
 
 
1731
  lines = text.splitlines()
1732
  out = []
1733
  i = 0
 
1735
  while i < len(lines):
1736
  line = lines[i].strip()
1737
 
 
1738
  if not _JB_SECTION_HDR.match(line):
1739
  out.append(lines[i])
1740
  i += 1
 
1743
  section_title = line
1744
  j = i + 1
1745
 
 
1746
  while j < len(lines) and not lines[j].strip():
1747
  j += 1
1748
 
 
1749
  if j >= len(lines) or not _JB_COL_HDR.match(lines[j].strip()):
1750
  out.append(lines[i])
1751
  i += 1
 
1754
  col_hdr_line = lines[j].strip()
1755
  data_start = j + 1
1756
 
 
1757
  first_data_idx = _find_first_data_row(lines, data_start)
1758
 
 
1759
  if first_data_idx is None or not _JB_DATE_DD.match(lines[first_data_idx].strip()):
1760
  out.append(lines[i])
1761
  i += 1
1762
  continue
1763
 
 
1764
  section_end = len(lines)
1765
  for k in range(data_start, len(lines)):
1766
  l = lines[k].strip()
 
1770
  section_end = k
1771
  break
1772
 
 
1773
  if _jb_section_has_table(lines, i, section_end):
1774
  out.append(lines[i])
1775
  i += 1
1776
  continue
1777
 
 
1778
  parts_upper = col_hdr_line.upper().split()
1779
  if len(parts_upper) == 2 and parts_upper[1] == "BALANCE":
1780
  header_cols = ["Date", "Balance"]
 
1801
  or row_line.startswith("Page:")):
1802
  break
1803
 
 
1804
  mb = _JB_BAL_ROW.match(row_line)
1805
  if mb and header_cols == ["Date", "Balance"]:
1806
  if current:
 
1809
  k += 1
1810
  continue
1811
 
 
1812
  mt = _JB_TXN_ROW.match(row_line)
1813
  if mt:
1814
  if current:
 
1822
  k += 1
1823
  continue
1824
 
 
1825
  if current and row_line:
1826
  desc_key = "Description" if "Description" in header_cols else "Number"
1827
  if desc_key in current:
 
1837
  if rows:
1838
  out.append(_jb_rows_to_html(header_cols, rows))
1839
 
1840
+ i = section_end
1841
  continue
1842
 
1843
  return "\n".join(out)
 
1847
  # --------------------------
1848
 
1849
  def _is_nfcu_document(pdf_path: str) -> bool:
 
 
 
 
1850
  try:
1851
  import pymupdf as fitz
1852
  doc = fitz.open(pdf_path)
 
1856
  text += "\n" + (doc[i].get_text() or "")
1857
  doc.close()
1858
  t = text.lower()
 
1859
  return (
1860
  ("statement of account" in t)
1861
  and ("access no." in t)
 
1867
 
1868
 
1869
  def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1870
  try:
1871
  import pymupdf as fitz
1872
 
 
1890
  if not is_nfcu:
1891
  return ""
1892
 
 
1893
  buckets = []
1894
  for w in words:
1895
  if len(w) < 5:
 
1951
  i += 1
1952
  continue
1953
 
 
1954
  if acct_re.match(text) and i != start_idx:
1955
  break
1956
  if any(text.startswith(s) for s in stop_markers):
1957
  break
 
1958
  upper_text = text.upper()
1959
  if (
1960
  "REMITTANCE RECEIVED" in upper_text
 
1991
  amount = mvals[-2]
1992
  balance = mvals[-1]
1993
  elif len(mvals) == 1:
 
1994
  balance = mvals[-1]
1995
 
1996
  pending = {
 
2004
  mvals = [tok for tok in toks if money_re.match(tok)]
2005
  desc_tokens = [tok for tok in toks if not money_re.match(tok)]
2006
 
 
 
2007
  upper = text.upper()
2008
  if pending["balance"] and (
2009
  "REMITTANCE" in upper
 
2015
  ):
2016
  break
2017
 
 
 
2018
  if (
2019
  ("BEGINNING BALANCE" in pending["desc"].upper() or "ENDING BALANCE" in pending["desc"].upper())
2020
  and not mvals
 
2030
  pending["balance"] = mvals[-1]
2031
  elif len(mvals) == 1 and not pending["balance"]:
2032
  pending["balance"] = mvals[-1]
 
2033
  i += 1
2034
 
2035
  if pending is not None:
2036
  rows.append(pending)
 
2037
  cleaned = [r for r in rows if r["date"] and r["desc"] and (r["amount"] or r["balance"])]
2038
  return cleaned, i
2039
 
 
2064
  i = i2
2065
  continue
2066
 
 
2067
  if text in ("Checking", "Savings"):
2068
  out.append(f"<h3>{text}</h3>")
2069
  elif text.startswith("Items Paid"):
 
2077
  return ""
2078
 
2079
  def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
 
 
 
 
 
2080
  try:
2081
  import pymupdf as fitz
2082
 
 
2090
  if not words:
2091
  return ""
2092
 
 
2093
  buckets = []
2094
  for w in words:
2095
  if len(w) < 5:
 
2132
  re.IGNORECASE,
2133
  )
2134
 
 
 
2135
  rows = []
2136
  pending_account = ""
2137
  account_name_re = re.compile(
 
2149
  t0 = toks[0].lower()
2150
  tl = text.lower()
2151
 
 
2152
  if (
2153
  "previous" in tl
2154
  or "deposits" in tl
 
2158
  ) and not any(money_re.match(t) for t in toks):
2159
  continue
2160
 
 
2161
  if account_name_re.match(text.strip()) and not any(money_re.match(t) for t in toks):
2162
  pending_account = text.strip()
2163
  continue
 
2165
  mvals = [t.replace("$", "") for t in toks if money_re.match(t)]
2166
  acct_tokens = [t for t in toks if acct_no_re.match(t)]
2167
 
 
2168
  if t0 == "totals" and len(mvals) >= 5:
2169
  rows.append(["Totals", mvals[0], mvals[1], mvals[2], mvals[3], mvals[4]])
2170
  continue
2171
 
 
2172
  if acct_tokens and len(mvals) >= 5:
2173
  acct = acct_tokens[0]
2174
  acct_label = (pending_account + " - " + acct).strip(" -") if pending_account else acct
 
2192
  return ""
2193
 
2194
  def _replace_nfcu_summary_table(page_md: str, summary_table_html: str) -> str:
 
 
 
 
2195
  if not page_md or not summary_table_html:
2196
  return page_md
2197
  table_pattern = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
2198
  lower = page_md.lower()
2199
  anchor = lower.find("summary of your deposit accounts")
2200
 
 
2201
  if anchor >= 0:
2202
  for m in table_pattern.finditer(page_md):
2203
  if m.start() > anchor:
2204
  old_table = m.group(0)
2205
  return page_md.replace(old_table, summary_table_html, 1)
2206
 
 
 
2207
  for m in table_pattern.finditer(page_md):
2208
  tbl = m.group(0)
2209
  p = TableGridParser()
 
2224
  return page_md
2225
 
2226
  def _split_fused_date_transaction_header(grid):
 
 
 
 
 
2227
  if not grid or len(grid) < 2:
2228
  return grid
2229
  ncols = max(len(r) for r in grid if isinstance(r, list))
 
2274
  return new_grid
2275
 
2276
  def _split_combined_summary_daily_balance_grid(grid):
 
 
 
 
 
 
 
2277
  if not grid or len(grid) < 4:
2278
  return None
2279
 
 
2307
  for d, a in zip(dates, amts):
2308
  daily_rows.append([d, a])
2309
 
 
2310
  if len(daily_rows) < 2:
2311
  return None
2312
 
 
2318
  return [summary, daily]
2319
 
2320
  def _normalize_double_desc_amount_header_table(grid):
 
 
 
 
 
 
 
2321
  if not grid or len(grid) < 2:
2322
  return grid
2323
 
2324
  def _hdr_tight(s):
2325
  return re.sub(r"\s+", "", str(s or "").strip().lower())
2326
 
 
2327
  expected5_t = ["date", "description", "amount", "description", "amount"]
2328
  expected4_t = ["date", "description", "amount", "description"]
2329
 
 
2373
  if not any(cells[:ncols]):
2374
  continue
2375
 
 
2376
  mpre = date_prefix_re.match(c0)
2377
  if mpre and _is_money(c1):
2378
  date_s = mpre.group(1).strip()
 
2386
  out.append([date_s, desc, c1])
2387
  continue
2388
 
 
2389
  if date_only_re.match(c0.strip()):
2390
  date_s = c0.strip()
2391
  desc = c1
 
2405
  out.append([date_s, desc, amt])
2406
  continue
2407
 
 
2408
  if not date_prefix_re.match(c0) and not date_only_re.match(c0.strip()) and _is_money(c1):
2409
  out.append(["", c0, c1])
2410
  continue
2411
 
 
2412
  out.append([c0, c1, c2])
2413
 
2414
  return out if len(out) >= 2 else grid
2415
 
2416
 
2417
  def _normalize_ucb_three_col_beginning_balance_fusion(grid):
 
 
 
 
 
 
 
2418
  if not grid or len(grid) < 2:
2419
  return grid
2420
 
 
2422
  return re.sub(r"\s+", "", str(s or "").strip().lower())
2423
 
2424
  h0 = [str(c or "").strip() for c in grid[0]]
 
2425
  if len(h0) < 3:
2426
  return grid
2427
  if _tight(h0[0]) != "date" or _tight(h0[1]) != "description" or _tight(h0[2]) != "amount":
 
2461
 
2462
 
2463
  def _fix_three_col_date_desc_amount_left_shift(grid):
 
 
 
 
 
 
2464
  if not grid or len(grid) < 2:
2465
  return grid
2466
 
 
2509
 
2510
 
2511
  def _normalize_fused_date_posted_amount_table(grid):
 
 
 
 
 
 
 
 
2512
  if not grid or len(grid) < 2:
2513
  return grid
2514
 
 
2541
  out.append(["Date", "Transaction Description", "Amount"])
2542
  continue
2543
  if ri < hdr_idx:
 
2544
  joined = " ".join(c for c in cells if c).strip()
2545
  out.append(["", joined, ""])
2546
  continue
 
2548
  c1 = cells[0].strip()
2549
  c2 = cells[1].strip() if len(cells) > 1 else ""
2550
 
 
2551
  lower_c1 = c1.lower()
2552
  if (
2553
  lower_c1.startswith("date posted transaction description")
 
2563
  date = parts[0]
2564
  desc = " ".join(parts[1:]).strip()
2565
 
 
 
2566
  amount = c2
2567
  if not amount:
2568
  money_cells = []
 
2573
  if money_cells:
2574
  amount = money_cells[-1]
2575
 
 
2576
  if not amount and desc:
2577
  m = re.search(r"(-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{2})-?)\s*$", desc)
2578
  if m:
 
2583
  continue
2584
  out.append([date, desc, amount])
2585
 
 
2586
  if len(out) < 2:
2587
  return grid
2588
  return out
2589
 
2590
  def normalize_html_tables(text: str) -> str:
 
 
 
 
 
 
 
 
 
 
 
2591
  if not text or "<table" not in text.lower():
2592
  return text
2593
 
 
2603
  p.feed(table_html)
2604
  grid = _build_grid(p.rows)
2605
 
 
2606
  if _is_junk_table(grid):
 
2607
  out.append("")
2608
+ last = m.end()
2609
  continue
2610
 
 
 
2611
  grid = _normalize_double_desc_amount_header_table(grid)
 
2612
  grid = _normalize_ucb_three_col_beginning_balance_fusion(grid)
 
2613
  grid = _fix_three_col_date_desc_amount_left_shift(grid)
2614
 
2615
  grid = _drop_truly_empty_columns(grid)
2616
  grid = _merge_blank_header_text_columns(grid)
2617
 
 
2618
  grid = _reconstruct_separator_rows(grid)
2619
 
 
2620
  grid, recovered_row = _clean_header_artifacts(grid)
2621
  if recovered_row is not None:
2622
  grid.insert(1, recovered_row)
2623
 
 
2624
  grid = _promote_misplaced_header_row(grid)
2625
 
 
2626
  grid = _split_fused_date_transaction_header(grid)
2627
 
 
 
2628
  grid = _normalize_fused_date_posted_amount_table(grid)
2629
 
 
2630
  grid = _split_fused_multicolumn_header(grid)
2631
 
 
2632
  grid = _normalize_daily_balance_table(grid)
2633
 
 
2634
  grid = _normalize_checks_paid_table(grid)
2635
 
 
 
2636
  grid = _merge_split_rows(grid)
2637
 
 
 
2638
  grid = _extract_fused_desc_amount(grid)
2639
 
 
2640
  grid = _repair_da3_garbled_amount_cells(grid)
2641
 
 
2642
  grid = _fix_fused_keyvalue_rows(grid)
 
2643
  grid = _dedupe_duplicate_rows_in_da3_table(grid)
 
2644
  split_tables = _split_combined_summary_daily_balance_grid(grid)
2645
  if split_tables:
2646
  out.append("\n\n".join(_grid_to_html(g) for g in split_tables if g))
 
2655
 
2656
 
2657
  def _transaction_row_dedupe_key(cells):
 
2658
  parts = [str(c or "").strip() for c in cells[:3]]
2659
  while len(parts) < 3:
2660
  parts.append("")
 
2662
  date_s = re.sub(r"\s+", "", date_s)
2663
  amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
2664
  desc_s = html.unescape(desc_s)
2665
+ desc_s = desc_s.replace("`", "'").replace("'", "'")
2666
  desc_s = re.sub(r"\s+", "", desc_s.lower())
2667
  return (date_s, desc_s, amt_s)
2668
 
2669
 
2670
  def _da3_loose_key(cells):
 
 
 
 
2671
  parts = [str(c or "").strip() for c in cells[:3]]
2672
  while len(parts) < 3:
2673
  parts.append("")
 
2689
 
2690
 
2691
  def _row_counter_loose_da3(g):
 
2692
  c = Counter()
2693
  if not g or len(g) < 2:
2694
  return c
 
2703
 
2704
 
2705
  def _overlap_ratio_counter(c_num, c_den):
 
2706
  tot = sum(c_num.values())
2707
  if tot == 0:
2708
  return 0.0
 
2734
 
2735
 
2736
  def _repair_da3_garbled_amount_cells(grid):
 
 
 
 
 
2737
  if not grid or len(grid) < 2 or not _is_da3_header(grid):
2738
  return grid
2739
 
 
2791
 
2792
 
2793
  def _dedupe_duplicate_rows_in_da3_table(grid):
 
 
 
 
 
2794
  if not grid or len(grid) < 2:
2795
  return grid
2796
  if not _is_da3_header(grid):
 
2827
 
2828
 
2829
  def _row_text_full_ucb(grid, r: int) -> str:
 
2830
  if not grid or r < 0 or r >= len(grid):
2831
  return ""
2832
  parts = [str(c or "").strip().lower() for c in grid[r]]
 
2841
 
2842
 
2843
  def _ucb_verify_electronic_credits_rows(grid, ec_start: int) -> bool:
 
2844
  if ec_start + 2 >= len(grid):
2845
  return False
2846
  t0 = _row_text_full_ucb(grid, ec_start)
 
2853
 
2854
 
2855
  def _ucb_split_grid_deposits_ec_ed(grid):
 
 
 
 
2856
  if not grid or len(grid) < 2:
2857
  return None
2858
  if not _is_da3_header(grid):
 
2880
 
2881
 
2882
  def _try_split_ucb_deposits_table_to_three(prefix: str, table_html: str):
 
 
 
 
2883
  try:
2884
  p = TableGridParser()
2885
  p.feed(table_html)
 
2907
 
2908
 
2909
  def _split_ucb_deposits_electronic_sections_html(text: str) -> str:
 
 
 
 
 
2910
  if not text or "deposits (continued)" not in text.lower():
2911
  return text
2912
 
 
2922
 
2923
 
2924
  def _parse_da3_grid_from_table_html(table_html: str):
 
2925
  try:
2926
  p = TableGridParser()
2927
  p.feed(table_html)
 
2934
 
2935
 
2936
  def _strip_orphan_continued_da3_flatline(text: str) -> str:
 
 
 
 
 
2937
  if not text or "(continued)" not in text.lower():
2938
  return text
2939
  lines = text.splitlines(keepends=True)
 
2956
 
2957
 
2958
  def _strip_standalone_continued_banner_line(text: str) -> str:
 
 
 
 
2959
  if not text or "(continued)" not in text.lower():
2960
  return text
2961
  lines = text.splitlines(keepends=True)
 
2983
 
2984
 
2985
  def _patch_da3_msft_g_ref_amounts_across_tables(text: str) -> str:
 
 
 
 
2986
  if not text or "<table" not in text.lower():
2987
  return text
2988
  if "microsoft#" not in text.lower():
 
3071
 
3072
 
3073
  def _trim_adjacent_da3_suffix_duplicating_next_table_prefix(text: str) -> str:
 
 
 
 
 
 
3074
  if not text or "<table" not in text.lower():
3075
  return text
3076
 
 
3123
 
3124
 
3125
  def _dedupe_subset_transaction_tables_html(text: str) -> str:
 
 
 
 
 
 
 
 
 
 
 
 
 
3126
  if not text or "<table" not in text.lower():
3127
  return text
3128
 
 
3177
  if not sj:
3178
  continue
3179
 
 
3180
  if sj <= si and len(sj) < len(si):
3181
  drop_idx.add(j)
3182
  continue
3183
 
 
3184
  if si <= sj and len(si) < len(sj):
3185
  drop_idx.add(i)
3186
  break
 
3190
  break
3191
 
3192
  inter = si & sj
 
3193
  ri = len(inter) / len(si) if si else 0.0
3194
  rj = len(inter) / len(sj) if sj else 0.0
3195
  if ri >= 0.88 and len(si) <= len(sj):
 
3199
  drop_idx.add(j)
3200
  continue
3201
 
 
3202
  ci = _row_counter_loose_da3(gi)
3203
  cj = _row_counter_loose_da3(gj)
3204
  ni, nj = sum(ci.values()), sum(cj.values())
3205
  if ni >= 1 and nj >= 1:
3206
  oi_j = _overlap_ratio_counter(ci, cj)
3207
  oj_i = _overlap_ratio_counter(cj, ci)
 
3208
  if (
3209
  nj >= 8
3210
  and 3 <= ni <= 14
 
3256
 
3257
 
3258
  def _da3_row_credit_vs_debit_category(desc: str) -> str:
 
 
 
 
3259
  d = (desc or "").lower()
3260
  if "debit card return" in d:
3261
  return "credit"
 
3269
  return "credit"
3270
  if "lrm claims" in d or ("claims" in d and "adj" in d):
3271
  return "credit"
 
3272
  if "adp tax" in d and "customer" in d:
3273
  return "credit"
3274
  if "debit card" in d:
 
3291
 
3292
 
3293
  def _da3_table_credit_debit_fractions(grid):
 
3294
  deb = cre = unk = 0
3295
  if not grid or len(grid) < 2:
3296
  return 0.0, 0.0, 0.0, 0
 
3316
 
3317
 
3318
  def _drop_da3_misplaced_debit_after_deposit_heading_tables(text: str) -> str:
 
 
 
 
 
 
3319
  if not text or "<table" not in text.lower():
3320
  return text
3321
  if "deposit" not in text.lower():
 
3347
  if n1 < 5 or n2 < 10:
3348
  pos = m.start() + 1
3349
  continue
 
3350
  credit_like_first = cr1 + 0.42 * unk1
3351
  if credit_like_first < 0.30 and credit_like_first <= db1 + 0.05:
3352
  pos = m.start() + 1
 
3364
  return text
3365
 
3366
 
3367
+ # --------------------------
3368
+ # Fix 28: Strip credit-row prefix from debit DA3 table
3369
+ # --------------------------
3370
+
3371
+ # Keywords that identify a section heading as debit/withdrawal-oriented.
3372
+ _DEBIT_HEADING_KEYWORDS = re.compile(
3373
+ r"\b(withdrawal|withdrawals|debit|debits|charges?|subtraction|subtractions|"
3374
+ r"payments?\s+made|money\s+out|paid\s+out|electronic\s+debit)\b",
3375
+ re.IGNORECASE,
3376
+ )
3377
+ # Keywords that identify a section heading as credit/deposit-oriented.
3378
+ _CREDIT_HEADING_KEYWORDS = re.compile(
3379
+ r"\b(deposit|deposits|credit|credits|addition|additions|interest\s+paid|"
3380
+ r"money\s+in|paid\s+in|incoming|received)\b",
3381
+ re.IGNORECASE,
3382
+ )
3383
+ # How many characters before a table start to scan for a section heading.
3384
+ _HEADING_SCAN_WINDOW = 600
3385
+
3386
+
3387
+ def _get_heading_before_table(text: str, table_start: int) -> str:
3388
+ """
3389
+ Return the last non-trivial text line in the _HEADING_SCAN_WINDOW chars
3390
+ that precede `table_start`. HTML tags are stripped before scanning so that
3391
+ centered <div> headings are picked up correctly.
3392
+ """
3393
+ window = text[max(0, table_start - _HEADING_SCAN_WINDOW): table_start]
3394
+ clean = re.sub(r"<[^>]+>", " ", window)
3395
+ lines = [ln.strip() for ln in clean.splitlines() if ln.strip()]
3396
+ for line in reversed(lines):
3397
+ if len(line) > 3 and "---" not in line:
3398
+ return line
3399
+ return ""
3400
+
3401
+
3402
+ def _strip_credits_prefix_from_debit_da3_table(text: str) -> str:
3403
+ """
3404
+ Fix 28: When OCR prepends the rows of a preceding credits/deposits DA3 table
3405
+ onto the top of a debits/withdrawals DA3 table, strip that prefix.
3406
+ Safety guards (ALL must pass):
3407
+ 1. Both tables must be valid DA3 (Date|Description|Amount header).
3408
+ 2. Target (later) table must follow a debit/withdrawal heading.
3409
+ 3. Source (earlier) table must follow a credit/deposit heading.
3410
+ 4. Only the contiguous leading prefix of matching rows is removed.
3411
+ 5. At least 1 data row must remain in the target after stripping.
3412
+ 6. Strip count must be >= 1.
3413
+ 7. Entire function wrapped in try/except — returns input unchanged on error.
3414
+ """
3415
+ if not text or "<table" not in text.lower():
3416
+ return text
3417
+ try:
3418
+ pattern = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
3419
+ matches = list(pattern.finditer(text))
3420
+ if len(matches) < 2:
3421
+ return text
3422
+
3423
+ # Collect all DA3 tables with their heading context.
3424
+ da3_tables = []
3425
+ for m in matches:
3426
+ g = _parse_da3_grid_from_table_html(m.group(0))
3427
+ if not g or not _is_da3_header(g):
3428
+ continue
3429
+ heading = _get_heading_before_table(text, m.start())
3430
+ da3_tables.append({"match": m, "grid": g, "heading": heading})
3431
+
3432
+ if len(da3_tables) < 2:
3433
+ return text
3434
+
3435
+ def _row_keys(grid):
3436
+ keys = set()
3437
+ for row in grid[1:]:
3438
+ cells = [str(c or "").strip() for c in row]
3439
+ if any(cells):
3440
+ keys.add(_transaction_row_dedupe_key(cells))
3441
+ return keys
3442
+
3443
+ # For each DA3 table preceded by a debit heading, look backward for the
3444
+ # nearest DA3 table preceded by a credit heading, and measure how many
3445
+ # of the debit table's leading rows appear in that credit table.
3446
+ drop_map = {} # table match start -> number of rows to strip
3447
+ for i in range(1, len(da3_tables)):
3448
+ ti = da3_tables[i]
3449
+ hi = ti["heading"].lower()
3450
+ if not _DEBIT_HEADING_KEYWORDS.search(hi):
3451
+ continue
3452
+ gi = ti["grid"]
3453
+ data_i = gi[1:]
3454
+ if len(data_i) < 2:
3455
+ continue
3456
+ for j in range(i - 1, -1, -1):
3457
+ tj = da3_tables[j]
3458
+ hj = tj["heading"].lower()
3459
+ if not _CREDIT_HEADING_KEYWORDS.search(hj):
3460
+ continue
3461
+ gj = tj["grid"]
3462
+ keys_j = _row_keys(gj)
3463
+ if not keys_j:
3464
+ continue
3465
+ # Count the contiguous leading rows of data_i that are in keys_j.
3466
+ strip_count = 0
3467
+ for row in data_i:
3468
+ cells = [str(c or "").strip() for c in row]
3469
+ k = _transaction_row_dedupe_key(cells)
3470
+ if k in keys_j:
3471
+ strip_count += 1
3472
+ else:
3473
+ break
3474
+ if strip_count < 1:
3475
+ continue
3476
+ # Guard: at least 1 data row must survive.
3477
+ if len(data_i) - strip_count < 1:
3478
+ continue
3479
+ drop_map[ti["match"].start()] = strip_count
3480
+ log.info(
3481
+ "Fix 28: stripped %d leading credit row(s) from debit table at offset %d",
3482
+ strip_count, ti["match"].start(),
3483
+ )
3484
+ break # found nearest credit table — stop looking further back
3485
+
3486
+ if not drop_map:
3487
+ return text
3488
+
3489
+ # Apply the strip.
3490
+ out_chunks = []
3491
+ last = 0
3492
+ for m in matches:
3493
+ out_chunks.append(text[last: m.start()])
3494
+ if m.start() in drop_map:
3495
+ strip_n = drop_map[m.start()]
3496
+ g = _parse_da3_grid_from_table_html(m.group(0))
3497
+ if g and _is_da3_header(g) and len(g) - 1 - strip_n >= 1:
3498
+ new_grid = [g[0]] + g[1 + strip_n:]
3499
+ out_chunks.append(_grid_to_html(new_grid))
3500
+ else:
3501
+ out_chunks.append(m.group(0))
3502
+ else:
3503
+ out_chunks.append(m.group(0))
3504
+ last = m.end()
3505
+ out_chunks.append(text[last:])
3506
+ return "".join(out_chunks)
3507
+
3508
+ except Exception as e:
3509
+ log.warning("_strip_credits_prefix_from_debit_da3_table failed: %s", e)
3510
+ return text
3511
+
3512
+
3513
  def stabilize_tables_and_text(page_md: str) -> str:
3514
  if not page_md:
3515
  return page_md
 
3534
  stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
3535
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
3536
  stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
3537
+ stabilized = _strip_credits_prefix_from_debit_da3_table(stabilized) # Fix 28
3538
  stabilized = _strip_orphan_continued_da3_flatline(stabilized)
3539
  stabilized = _strip_standalone_continued_banner_line(stabilized)
3540
  return close_unclosed_html(stabilized)
 
3639
 
3640
  parts = []
3641
 
 
 
 
 
3642
  hdr = ""
3643
  if is_pdf:
3644
  hdr = extract_zone_text_pdf(path, page_num, 0, he)
 
3649
  if hdr and hdr.strip():
3650
  parts.append(normalize_html_tables(fix_account_number(normalize_money_glyphs(hdr.strip()))))
3651
 
 
3652
  if page_md and page_md.strip():
3653
  stabilized = stabilize_tables_and_text(page_md.strip())
 
 
3654
  if is_pdf and not is_nfcu_doc:
3655
  stabilized = _patch_ocr_with_textlayer(stabilized, path, page_num)
3656
  stabilized = _strip_orphan_continued_da3_flatline(stabilized)
3657
  stabilized = _strip_standalone_continued_banner_line(stabilized)
 
 
3658
  stabilized = normalize_html_tables(stabilized)
3659
  stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
3660
  stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
3661
  stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
3662
  stabilized = _dedupe_subset_transaction_tables_html(stabilized)
3663
  stabilized = _drop_da3_misplaced_debit_after_deposit_heading_tables(stabilized)
3664
+ stabilized = _strip_credits_prefix_from_debit_da3_table(stabilized) # Fix 28
3665
  stabilized = _strip_standalone_continued_banner_line(stabilized)
3666
  elif is_pdf and is_nfcu_doc:
 
 
3667
  nfcu_summary = _extract_nfcu_summary_table(path, page_num)
3668
  if nfcu_summary:
3669
  stabilized = _replace_nfcu_summary_table(stabilized, nfcu_summary)
3670
 
 
 
3671
  if is_pdf:
3672
  try:
3673
  needs_checks = 'class="jb-summary-checks"' not in stabilized
3674
  needs_dab = "daily account balance" not in stabilized.lower()
3675
  if needs_checks or needs_dab:
 
3676
  _txt4d = ""
3677
  try:
3678
  from pdfminer.high_level import extract_text as _pm_extract
 
3692
  if jb_extra:
3693
  stabilized = stabilized.rstrip() + "\n\n" + jb_extra
3694
  except Exception:
3695
+ pass
3696
 
3697
  parts.append(stabilized)
3698
 
 
 
3699
  if ENABLE_FOOTER_OCR and page_num < len(page_images):
3700
  ftr = ""
3701
  if is_pdf:
 
3707
  if ftr and ftr.strip():
3708
  ftr_clean = normalize_money_glyphs(ftr.strip())
3709
 
 
3710
  ftr_first_line = next(
3711
  (ln.strip().lower() for ln in ftr_clean.splitlines() if ln.strip()),
3712
  ""
 
3715
  ftr_first_line in part.lower() for part in parts
3716
  )
3717
 
 
 
 
 
 
3718
  _footer_date_re = re.compile(r"\b\d{1,2}[-/]\d{2}\b")
3719
  _footer_amt_re = re.compile(r"\b\d{1,3}(?:,\d{3})*\.\d{2}\b")
3720
  _date_hits = len(_footer_date_re.findall(ftr_clean))
 
3728
  all_pages.append("\n\n".join(parts))
3729
 
3730
  merged = "\n\n---page-separator---\n\n".join(all_pages) if all_pages else "(No content)"
 
3731
  if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
3732
  merged = _strip_orphan_continued_da3_flatline(merged)
3733
  merged = _strip_standalone_continued_banner_line(merged)
 
3736
  merged = _patch_da3_msft_g_ref_amounts_across_tables(merged)
3737
  merged = _dedupe_subset_transaction_tables_html(merged)
3738
  merged = _drop_da3_misplaced_debit_after_deposit_heading_tables(merged)
3739
+ merged = _strip_credits_prefix_from_debit_da3_table(merged) # Fix 28
3740
  merged = _strip_orphan_continued_da3_flatline(merged)
3741
  merged = _strip_standalone_continued_banner_line(merged)
3742
  return merged