rehan953 commited on
Commit
5bef6c6
·
verified ·
1 Parent(s): b56c268

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +213 -173
app.py CHANGED
@@ -594,15 +594,12 @@ def _extract_fused_header_artifacts(header_row):
594
  """
595
  Generic detector for the OCR artifact where the first data row of a table
596
  gets fused into the header cells during OCR.
597
-
598
  The artifact pattern (BOTH signals must fire simultaneously):
599
  - Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
600
  e.g. "DESCRIPTIONBeginning Balance" "NARRATIONOpening Balance"
601
  - Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
602
  e.g. "BALANCE$3,447.10" "AMOUNT 5,000.00"
603
-
604
  Two-signal guard prevents false positives on clean PDFs from any bank.
605
-
606
  Returns:
607
  (cleaned_header : list[str], recovered_row : list[str] | None)
608
  """
@@ -707,15 +704,12 @@ def _promote_misplaced_header_row(grid):
707
  """
708
  Detects and fixes the OCR artifact where real column headers land in a
709
  <td> data row instead of the <th> header row.
710
-
711
  Two scenarios handled:
712
-
713
  Scenario A — Simple misplaced header (e.g. TD Bank):
714
  <th>ACCOUNT ACTIVITY</th> ← section title, not a real header
715
  <td>DATE DESCRIPTION</td> ... ← real column headers in a data row
716
  <td>01/05 ...</td> ← data
717
  → Promote the keyword row to header, discard the section title rows above.
718
-
719
  Scenario B — Metadata header + misplaced column headers (e.g. Citi page 1):
720
  <th>208479667</th> <th>Beginning Balance:$20.48 Ending Balance:$3.46</th>
721
  <td>Date Description</td> <td>Debits</td> <td>Credits</td> <td>Balance</td>
@@ -725,7 +719,6 @@ def _promote_misplaced_header_row(grid):
725
  → Preserve metadata row cells as a plain-text prefix OUTSIDE the table
726
  by embedding them as a leading data row with colspan (kept for reference).
727
  → Split any "Date Description" fused cells in data rows.
728
-
729
  Guard conditions (no-op if not met — safe for all other PDFs):
730
  - Current header keyword score < threshold.
731
  - A data row within the first 5 rows has keyword score >= threshold.
@@ -821,10 +814,8 @@ def _fix_fused_keyvalue_rows(grid):
821
  Fix rows where label+value are fused into the first cell with all other
822
  cells empty — common in Interest Summary / Fee Summary sections appended
823
  to a transaction table by OCR.
824
-
825
  e.g. "Beginning Interest Rate0.00%" → label="Beginning Interest Rate" value="0.00%"
826
  "Number of days in this Period31" → label="..." value="31"
827
-
828
  Guard conditions (no-op if not met — safe for all other tables):
829
  - All cells except first must be empty.
830
  - A non-space character must immediately precede the digit run (fused).
@@ -912,11 +903,9 @@ def _fix_fused_keyvalue_rows(grid):
912
  def _extract_textlayer_rows(pdf_path: str, page_num: int):
913
  """
914
  Extract structured transaction rows from the PDF text layer using pymupdf.
915
-
916
  Returns list of {"date": str, "desc": str, "amount": str} dicts, or []
917
  if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
918
  transaction rows are found (guards against non-transaction pages).
919
-
920
  Supports multiple date formats:
921
  - Numeric: MM/DD, MM/DD/YY, MM/DD/YYYY (Chase, TD, BoA, Citi)
922
  - Month-name: Jan 16, Feb 7, Mar 05 (Capital One, Amex)
@@ -1047,7 +1036,6 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
1047
  log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
1048
  return []
1049
 
1050
-
1051
  def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1052
  """
1053
  Johnson Bank: when pdfminer column extraction fails (e.g. pymupdf line text or
@@ -1081,18 +1069,15 @@ def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1081
  out.append(t)
1082
  return out
1083
 
1084
-
1085
  def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
1086
  """
1087
  Parse Johnson Bank Checks and Daily Account Balance from pdfminer column-separated
1088
  page text and return as HTML tables.
1089
-
1090
  pdfminer outputs multi-column sections as separate column lists:
1091
  Date Number Amount
1092
  04-14 3259 45.72
1093
  becomes:
1094
  "Date\n04-14\n04-09\n\nNumber\n3259\n3260\n\nAmount\n45.72\n1,628.00"
1095
-
1096
  Uses pdfminer (always available) not pymupdf.
1097
  """
1098
  if not text_layer:
@@ -1211,7 +1196,6 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1211
  """
1212
  Compare OCR table rows against PDF text-layer rows and inject any rows
1213
  the OCR model silently dropped (typically identical consecutive rows).
1214
-
1215
  Guard conditions (all must pass for any injection):
1216
  1. PDF text layer must exist and yield >= 2 transaction rows.
1217
  2. The OCR table must have DATE + DESCRIPTION + AMOUNT columns.
@@ -1456,16 +1440,13 @@ def _reconstruct_separator_rows(grid):
1456
  """
1457
  Detect and reconstruct mid-table separator / label rows that OCR has
1458
  incorrectly split across columns.
1459
-
1460
  The artifact (Bank of America and similar):
1461
  A section label like "Card account # XXXX XXXX XXXX 2889" sits between
1462
  transaction rows as a full-width label. OCR splits the trailing digits
1463
  across columns because they are right-aligned, producing variations like:
1464
-
1465
  2-cell split: ["Card account # XXXX XXXX XXXX 2", "", "889"]
1466
  3-cell split: ["Card account # XXXX XXXX XXXX", "2", "889"]
1467
  clean 1-cell: ["Card account # XXXX XXXX XXXX 2889", "", ""]
1468
-
1469
  Detection criteria (ALL must hold — guards safe for all other PDFs):
1470
  1. First cell is NOT a date token (transaction rows always start with date).
1471
  2. First cell contains letters (it is a label, not a bare number).
@@ -1475,7 +1456,6 @@ def _reconstruct_separator_rows(grid):
1475
  — these are the split-off tails of the account/card number.
1476
  4. There must be at least one non-empty cell after the first (to detect
1477
  the split; single-cell rows are also handled as clean label rows).
1478
-
1479
  The fix:
1480
  Concatenate all non-empty cells in order, place the result in col 0,
1481
  blank all other cells.
@@ -1551,19 +1531,15 @@ def _merge_split_rows(grid):
1551
  """
1552
  Fix OCR artifact where a transaction row is split across two <tr> rows
1553
  because the description wrapped to a second line in the source PDF.
1554
-
1555
  Two patterns detected (Hardin County Bank and similar monospace statements):
1556
-
1557
  Pattern A — description + empty continuation:
1558
  Row N: [desc, '', '', '', ''] ← description, no date/money
1559
  Row N+1: ['', debit, credit, date, bal] ← money/date, no description
1560
  → Merge into: [desc, debit, credit, date, bal]
1561
-
1562
  Pattern B — description + text continuation + money:
1563
  Row N: [desc, '', '', '', ''] ← first line of description
1564
  Row N+1: [desc_cont, debit, credit, date, bal] ← continuation + money
1565
  → Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
1566
-
1567
  Guard conditions (no-op unless both rows match the pattern):
1568
  - Row N must have non-empty col 0 (description).
1569
  - Row N must have empty date column (col 3 or wherever DATE is).
@@ -1655,11 +1631,9 @@ def _extract_fused_desc_amount(grid):
1655
  """
1656
  Fix OCR artifact where a trailing money amount gets fused into the
1657
  description cell instead of its own debit/credit column.
1658
-
1659
  Example (Hardin County Bank):
1660
  OCR: ['CHASE CREDIT CRD EPAY 8319249882 500.00', '', '', '04/11/25', '16,699.04']
1661
  Fix: ['CHASE CREDIT CRD EPAY 8319249882', '500.00', '', '04/11/25', '16,699.04']
1662
-
1663
  Guard conditions (all must hold):
1664
  - Description ends with a space + money amount (NNN.NN or N,NNN.NN).
1665
  - ALL debit AND credit columns are empty for this row.
@@ -1718,7 +1692,6 @@ def _is_junk_table(grid):
1718
  """
1719
  Detect and suppress known junk tables that are OCR artifacts from
1720
  page headers/footers, not real transaction data.
1721
-
1722
  Current patterns:
1723
  1. CUSTOMER INFORMATION fragment tables (First Horizon Bank):
1724
  Single-column table with header "CUSTOMER INFORMATION" and
@@ -1741,15 +1714,12 @@ def _is_junk_table(grid):
1741
  def _split_fused_multicolumn_header(grid):
1742
  """
1743
  Fix OCR artifact where multiple column headers are fused into one <th> cell.
1744
-
1745
  Example (First Horizon Bank alternating pages):
1746
  Fused: ['DATE DESCRIPTION CARD #', 'DEPOSIT', 'WITHDRAWAL']
1747
  Correct: ['DATE', 'DESCRIPTION', 'DEPOSIT', 'WITHDRAWAL', 'CARD #']
1748
-
1749
  The data rows also have date + description + card# all in col 0:
1750
  Fused: ['01/16 PURCHASE - UBER TRIP... 4919', '', '$21.02']
1751
  Correct: ['01/16', 'PURCHASE - UBER TRIP...', '', '$21.02', '4919']
1752
-
1753
  Detection: first header cell contains 2+ keyword terms separated by spaces,
1754
  AND remaining header cells are valid money column keywords.
1755
  """
@@ -1862,7 +1832,6 @@ def _normalize_daily_balance_table(grid):
1862
  - OCR produces N DATE columns followed by N BALANCE columns
1863
  instead of interleaved DATE|BALANCE|DATE|BALANCE pairs
1864
  - Multiple dates are stacked in one cell "01/02\n01/03\n01/04\n01/05"
1865
-
1866
  Reorders columns and expands stacked date cells into proper rows.
1867
  Only fires when ALL of these hold:
1868
  - Header has exactly 2N columns where N >= 2
@@ -1940,9 +1909,7 @@ def _normalize_checks_paid_table(grid):
1940
  - Remaining headers = "AMOUNT AMOUNT AMOUNT"
1941
  - Data col0 = "01/24 4701 01/18 4704 * 01/18 916831 *" (all groups fused)
1942
  - Remaining data cells = amount values
1943
-
1944
  Expands to proper: DATE | CHECK # | AMOUNT | DATE | CHECK # | AMOUNT | ...
1945
-
1946
  Detection: first header cell matches pattern (DATE CHECK #)+
1947
  and remaining headers are all AMOUNT.
1948
  """
@@ -2191,157 +2158,224 @@ def convert_plaintext_bank_sections(text: str) -> str:
2191
  return "\n".join(out)
2192
 
2193
  # --------------------------
2194
- # Navy Federal Summary Table extraction (Fix NF-1)
2195
  # --------------------------
2196
 
2197
- def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
2198
  """
2199
- Extract Navy Federal "Summary of your deposit accounts" table from the PDF
2200
- text layer using word coordinates.
2201
-
2202
- GLM-OCR fails on this table because the five data columns are spread very
2203
- wide across the page with no visible borders.
2204
-
2205
- Uses word x/y coordinates to reconstruct the table spatially.
2206
- Tries pymupdf first (HuggingFace), falls back to pdfplumber (local).
2207
-
2208
- Detection guards (safe no-op for all other banks):
2209
- 1. Word "Summary" in top 55% of page.
2210
- 2. Words "deposit" or "accounts" within 15pt of it.
2211
-
2212
- Returns HTML table string, or "" if not found/applicable.
2213
  """
2214
  try:
2215
- from collections import defaultdict as _dd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2216
 
2217
- words = None # list of (x0, top, text)
2218
- page_h = 1000
2219
 
2220
- # Try pymupdf first (HuggingFace environment)
2221
- try:
2222
- import pymupdf as _fitz_s
2223
- _doc_s = _fitz_s.open(pdf_path)
2224
- if page_num >= len(_doc_s):
2225
- _doc_s.close()
2226
- return ""
2227
- _pg_s = _doc_s[page_num]
2228
- page_h = _pg_s.rect.height
2229
- words = [(w[0], w[1], w[4]) for w in _pg_s.get_text("words")]
2230
- _doc_s.close()
2231
- except Exception:
2232
- pass
 
 
 
 
 
2233
 
2234
- # Fallback: pdfplumber (local environment)
2235
- if not words:
2236
- try:
2237
- import pdfplumber as _plumber_s
2238
- with _plumber_s.open(pdf_path) as _pdf_s:
2239
- if page_num >= len(_pdf_s.pages):
2240
- return ""
2241
- page_h = _pdf_s.pages[page_num].height
2242
- words = [(w["x0"], w["top"], w["text"])
2243
- for w in _pdf_s.pages[page_num].extract_words()]
2244
- except Exception:
2245
- pass
2246
 
2247
- if not words:
2248
  return ""
2249
 
2250
- # Guard 1: "Summary" word in top 55% of page
2251
- summary_word = next(
2252
- (w for w in words if w[2].lower() == "summary" and w[1] < page_h * 0.55),
2253
- None
2254
  )
2255
- if not summary_word:
2256
  return ""
2257
 
2258
- # Guard 2: "deposit" or "accounts" within 15pt
2259
- nearby = [w[2].lower() for w in words if abs(w[1] - summary_word[1]) < 15]
2260
- if "deposit" not in nearby and "accounts" not in nearby:
2261
- return ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2262
 
2263
- # Band: 15-115pt below the heading
2264
- y_start = summary_word[1] + 15
2265
- y_end = summary_word[1] + 115
2266
- band = [w for w in words if y_start <= w[1] <= y_end]
2267
- if not band:
2268
- return ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2269
 
2270
- # Group into lines by y-bucket (5pt tolerance)
2271
- lines_by_y = _dd(list)
2272
- for w in band:
2273
- lines_by_y[round(w[1] / 5) * 5].append(w)
2274
- rows_raw = [sorted(lines_by_y[y], key=lambda w: w[0])
2275
- for y in sorted(lines_by_y)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2276
 
2277
- if len(rows_raw) < 3:
2278
- return ""
 
 
 
2279
 
2280
- # Column centers from first two header rows
2281
- header_xs = sorted(w[0] for r in rows_raw[:2] for w in r)
2282
- col_centers = []
2283
- cluster = [header_xs[0]]
2284
- for x in header_xs[1:]:
2285
- if x - cluster[-1] > 30:
2286
- col_centers.append(sum(cluster) / len(cluster))
2287
- cluster = [x]
2288
- else:
2289
- cluster.append(x)
2290
- col_centers.append(sum(cluster) / len(cluster))
2291
-
2292
- def _assign(x):
2293
- return min(range(len(col_centers)), key=lambda i: abs(col_centers[i] - x))
2294
-
2295
- # Build column labels from first two rows
2296
- col_labels = [""] * len(col_centers)
2297
- for r in rows_raw[:2]:
2298
- for w in r:
2299
- ci = _assign(w[0])
2300
- col_labels[ci] = (col_labels[ci] + " " + w[2]).strip()
2301
-
2302
- # Parse data rows
2303
- _MONEY = re.compile(r"^\$?-?[\d,]+\.\d{2}-?$")
2304
- merged, cur_name, cur_vals = [], [], {}
2305
-
2306
- for line in rows_raw[2:]:
2307
- texts = [w[2] for w in line]
2308
- has_money = any(_MONEY.match(t) for t in texts)
2309
- is_totals = any(w[2].lower() == "totals" for w in line)
2310
-
2311
- if has_money:
2312
- for w in line:
2313
- if not re.match(r"^\d{7,}$", w[2]) and w[2].lower() != "totals":
2314
- cur_vals[_assign(w[0])] = w[2]
2315
- acct_num = next((w[2] for w in line if re.match(r"^\d{7,}$", w[2])), "")
2316
- name = " ".join(cur_name)
2317
- if acct_num:
2318
- name = (name + " " + acct_num).strip()
2319
- if is_totals:
2320
- name = "Totals"
2321
- merged.append((name, dict(cur_vals)))
2322
- cur_name, cur_vals = [], {}
2323
- else:
2324
- cur_name.extend(texts)
2325
 
2326
- if not merged:
2327
- return ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2328
 
2329
- n = len(col_labels)
2330
- hdr = "<tr><th>Account</th>" + "".join(f"<th>{l}</th>" for l in col_labels) + "</tr>"
2331
- rows_html = [
2332
- "<tr><td>" + name + "</td>" +
2333
- "".join(f"<td>{vals.get(i, '')}</td>" for i in range(n)) +
2334
- "</tr>"
2335
- for name, vals in merged
2336
- ]
2337
- return (
2338
- "Summary of your deposit accounts\n"
2339
- "<table>\n" + hdr + "\n" +
2340
- "\n".join(rows_html) + "\n</table>"
2341
- )
2342
 
2343
  except Exception as e:
2344
- log.warning("_extract_nfcu_summary_table failed (page %d): %s", page_num, e)
2345
  return ""
2346
 
2347
  def normalize_html_tables(text: str) -> str:
@@ -2509,6 +2543,8 @@ def run_ocr(uploaded_file):
2509
  try:
2510
  path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
2511
  is_pdf = path.lower().endswith(".pdf")
 
 
2512
  parser = get_parser()
2513
 
2514
  page_heights = []
@@ -2538,6 +2574,21 @@ def run_ocr(uploaded_file):
2538
 
2539
  parts = []
2540
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2541
  # Header inclusion: PDF text -> band words -> OCR band
2542
  hdr = ""
2543
  if is_pdf:
@@ -2588,17 +2639,6 @@ def run_ocr(uploaded_file):
2588
 
2589
  parts.append(stabilized)
2590
 
2591
- # Fix NF-1: prepend Navy Federal summary table from text layer.
2592
- # Replaces the garbled OCR summary table with correct data.
2593
- # Safe no-op for all non-NFCU PDFs (returns "" when guards fail).
2594
- if is_pdf:
2595
- try:
2596
- nfcu_summary = _extract_nfcu_summary_table(path, page_num)
2597
- if nfcu_summary:
2598
- parts.insert(0, nfcu_summary)
2599
- except Exception:
2600
- pass
2601
-
2602
  # Footer extraction: PDF text layer -> band words -> OCR crop
2603
  # Deduplication guard prevents double-printing when body OCR already captured it.
2604
  if ENABLE_FOOTER_OCR and page_num < len(page_images):
 
594
  """
595
  Generic detector for the OCR artifact where the first data row of a table
596
  gets fused into the header cells during OCR.
 
597
  The artifact pattern (BOTH signals must fire simultaneously):
598
  - Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
599
  e.g. "DESCRIPTIONBeginning Balance" "NARRATIONOpening Balance"
600
  - Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
601
  e.g. "BALANCE$3,447.10" "AMOUNT 5,000.00"
 
602
  Two-signal guard prevents false positives on clean PDFs from any bank.
 
603
  Returns:
604
  (cleaned_header : list[str], recovered_row : list[str] | None)
605
  """
 
704
  """
705
  Detects and fixes the OCR artifact where real column headers land in a
706
  <td> data row instead of the <th> header row.
 
707
  Two scenarios handled:
 
708
  Scenario A — Simple misplaced header (e.g. TD Bank):
709
  <th>ACCOUNT ACTIVITY</th> ← section title, not a real header
710
  <td>DATE DESCRIPTION</td> ... ← real column headers in a data row
711
  <td>01/05 ...</td> ← data
712
  → Promote the keyword row to header, discard the section title rows above.
 
713
  Scenario B — Metadata header + misplaced column headers (e.g. Citi page 1):
714
  <th>208479667</th> <th>Beginning Balance:$20.48 Ending Balance:$3.46</th>
715
  <td>Date Description</td> <td>Debits</td> <td>Credits</td> <td>Balance</td>
 
719
  → Preserve metadata row cells as a plain-text prefix OUTSIDE the table
720
  by embedding them as a leading data row with colspan (kept for reference).
721
  → Split any "Date Description" fused cells in data rows.
 
722
  Guard conditions (no-op if not met — safe for all other PDFs):
723
  - Current header keyword score < threshold.
724
  - A data row within the first 5 rows has keyword score >= threshold.
 
814
  Fix rows where label+value are fused into the first cell with all other
815
  cells empty — common in Interest Summary / Fee Summary sections appended
816
  to a transaction table by OCR.
 
817
  e.g. "Beginning Interest Rate0.00%" → label="Beginning Interest Rate" value="0.00%"
818
  "Number of days in this Period31" → label="..." value="31"
 
819
  Guard conditions (no-op if not met — safe for all other tables):
820
  - All cells except first must be empty.
821
  - A non-space character must immediately precede the digit run (fused).
 
903
  def _extract_textlayer_rows(pdf_path: str, page_num: int):
904
  """
905
  Extract structured transaction rows from the PDF text layer using pymupdf.
 
906
  Returns list of {"date": str, "desc": str, "amount": str} dicts, or []
907
  if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
908
  transaction rows are found (guards against non-transaction pages).
 
909
  Supports multiple date formats:
910
  - Numeric: MM/DD, MM/DD/YY, MM/DD/YYYY (Chase, TD, BoA, Citi)
911
  - Month-name: Jan 16, Feb 7, Mar 05 (Capital One, Amex)
 
1036
  log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
1037
  return []
1038
 
 
1039
  def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1040
  """
1041
  Johnson Bank: when pdfminer column extraction fails (e.g. pymupdf line text or
 
1069
  out.append(t)
1070
  return out
1071
 
 
1072
  def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
1073
  """
1074
  Parse Johnson Bank Checks and Daily Account Balance from pdfminer column-separated
1075
  page text and return as HTML tables.
 
1076
  pdfminer outputs multi-column sections as separate column lists:
1077
  Date Number Amount
1078
  04-14 3259 45.72
1079
  becomes:
1080
  "Date\n04-14\n04-09\n\nNumber\n3259\n3260\n\nAmount\n45.72\n1,628.00"
 
1081
  Uses pdfminer (always available) not pymupdf.
1082
  """
1083
  if not text_layer:
 
1196
  """
1197
  Compare OCR table rows against PDF text-layer rows and inject any rows
1198
  the OCR model silently dropped (typically identical consecutive rows).
 
1199
  Guard conditions (all must pass for any injection):
1200
  1. PDF text layer must exist and yield >= 2 transaction rows.
1201
  2. The OCR table must have DATE + DESCRIPTION + AMOUNT columns.
 
1440
  """
1441
  Detect and reconstruct mid-table separator / label rows that OCR has
1442
  incorrectly split across columns.
 
1443
  The artifact (Bank of America and similar):
1444
  A section label like "Card account # XXXX XXXX XXXX 2889" sits between
1445
  transaction rows as a full-width label. OCR splits the trailing digits
1446
  across columns because they are right-aligned, producing variations like:
 
1447
  2-cell split: ["Card account # XXXX XXXX XXXX 2", "", "889"]
1448
  3-cell split: ["Card account # XXXX XXXX XXXX", "2", "889"]
1449
  clean 1-cell: ["Card account # XXXX XXXX XXXX 2889", "", ""]
 
1450
  Detection criteria (ALL must hold — guards safe for all other PDFs):
1451
  1. First cell is NOT a date token (transaction rows always start with date).
1452
  2. First cell contains letters (it is a label, not a bare number).
 
1456
  — these are the split-off tails of the account/card number.
1457
  4. There must be at least one non-empty cell after the first (to detect
1458
  the split; single-cell rows are also handled as clean label rows).
 
1459
  The fix:
1460
  Concatenate all non-empty cells in order, place the result in col 0,
1461
  blank all other cells.
 
1531
  """
1532
  Fix OCR artifact where a transaction row is split across two <tr> rows
1533
  because the description wrapped to a second line in the source PDF.
 
1534
  Two patterns detected (Hardin County Bank and similar monospace statements):
 
1535
  Pattern A — description + empty continuation:
1536
  Row N: [desc, '', '', '', ''] ← description, no date/money
1537
  Row N+1: ['', debit, credit, date, bal] ← money/date, no description
1538
  → Merge into: [desc, debit, credit, date, bal]
 
1539
  Pattern B — description + text continuation + money:
1540
  Row N: [desc, '', '', '', ''] ← first line of description
1541
  Row N+1: [desc_cont, debit, credit, date, bal] ← continuation + money
1542
  → Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
 
1543
  Guard conditions (no-op unless both rows match the pattern):
1544
  - Row N must have non-empty col 0 (description).
1545
  - Row N must have empty date column (col 3 or wherever DATE is).
 
1631
  """
1632
  Fix OCR artifact where a trailing money amount gets fused into the
1633
  description cell instead of its own debit/credit column.
 
1634
  Example (Hardin County Bank):
1635
  OCR: ['CHASE CREDIT CRD EPAY 8319249882 500.00', '', '', '04/11/25', '16,699.04']
1636
  Fix: ['CHASE CREDIT CRD EPAY 8319249882', '500.00', '', '04/11/25', '16,699.04']
 
1637
  Guard conditions (all must hold):
1638
  - Description ends with a space + money amount (NNN.NN or N,NNN.NN).
1639
  - ALL debit AND credit columns are empty for this row.
 
1692
  """
1693
  Detect and suppress known junk tables that are OCR artifacts from
1694
  page headers/footers, not real transaction data.
 
1695
  Current patterns:
1696
  1. CUSTOMER INFORMATION fragment tables (First Horizon Bank):
1697
  Single-column table with header "CUSTOMER INFORMATION" and
 
1714
  def _split_fused_multicolumn_header(grid):
1715
  """
1716
  Fix OCR artifact where multiple column headers are fused into one <th> cell.
 
1717
  Example (First Horizon Bank alternating pages):
1718
  Fused: ['DATE DESCRIPTION CARD #', 'DEPOSIT', 'WITHDRAWAL']
1719
  Correct: ['DATE', 'DESCRIPTION', 'DEPOSIT', 'WITHDRAWAL', 'CARD #']
 
1720
  The data rows also have date + description + card# all in col 0:
1721
  Fused: ['01/16 PURCHASE - UBER TRIP... 4919', '', '$21.02']
1722
  Correct: ['01/16', 'PURCHASE - UBER TRIP...', '', '$21.02', '4919']
 
1723
  Detection: first header cell contains 2+ keyword terms separated by spaces,
1724
  AND remaining header cells are valid money column keywords.
1725
  """
 
1832
  - OCR produces N DATE columns followed by N BALANCE columns
1833
  instead of interleaved DATE|BALANCE|DATE|BALANCE pairs
1834
  - Multiple dates are stacked in one cell "01/02\n01/03\n01/04\n01/05"
 
1835
  Reorders columns and expands stacked date cells into proper rows.
1836
  Only fires when ALL of these hold:
1837
  - Header has exactly 2N columns where N >= 2
 
1909
  - Remaining headers = "AMOUNT AMOUNT AMOUNT"
1910
  - Data col0 = "01/24 4701 01/18 4704 * 01/18 916831 *" (all groups fused)
1911
  - Remaining data cells = amount values
 
1912
  Expands to proper: DATE | CHECK # | AMOUNT | DATE | CHECK # | AMOUNT | ...
 
1913
  Detection: first header cell matches pattern (DATE CHECK #)+
1914
  and remaining headers are all AMOUNT.
1915
  """
 
2158
  return "\n".join(out)
2159
 
2160
  # --------------------------
2161
+ # Navy Federal full-page text-layer parser (Fix NF-1)
2162
  # --------------------------
2163
 
2164
+ def _is_nfcu_document(pdf_path: str) -> bool:
2165
  """
2166
+ Strict detector for Navy Federal statements.
2167
+ Used to gate NF-specific parsing so other banks are unaffected.
 
 
 
 
 
 
 
 
 
 
 
 
2168
  """
2169
  try:
2170
+ import pymupdf as fitz
2171
+ doc = fitz.open(pdf_path)
2172
+ pages_to_check = min(2, len(doc))
2173
+ text = ""
2174
+ for i in range(pages_to_check):
2175
+ text += "\n" + (doc[i].get_text() or "")
2176
+ doc.close()
2177
+ t = text.lower()
2178
+ # Accept either explicit Navy branding OR the NFCU statement signature fields.
2179
+ has_brand = ("navy federal" in t or "navyfederal.org" in t)
2180
+ has_signature_fields = (
2181
+ ("statement of account" in t)
2182
+ and ("statement period" in t)
2183
+ and ("access no." in t)
2184
+ and ("routing number" in t)
2185
+ )
2186
+ return has_brand or has_signature_fields
2187
+ except Exception:
2188
+ return False
2189
 
 
 
2190
 
2191
+ def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2192
+ """
2193
+ Full text-layer parser for Navy Federal Credit Union statements.
2194
+ GLM-OCR fails on NFCU pages because:
2195
+ - The summary table has 5 wide columns with no visible borders
2196
+ - Transaction tables have Amount($) and Balance($) spatially far right
2197
+ - Multi-line descriptions (wrapped lines) confuse OCR table detection
2198
+ This function reads the PDF text layer directly via pdfplumber and produces
2199
+ correct, complete HTML for every section: summary table, transaction tables,
2200
+ items paid, savings, disclosures.
2201
+ Detection guards (all must pass — safe no-op for all other banks):
2202
+ 1. Page text must contain 'Access No.' AND 'Statement of Account'
2203
+ (unique to Navy Federal statement format)
2204
+ 2. pdfplumber must return extractable text (not a scanned page)
2205
+ Returns complete HTML string for the page, or "" if not NFCU / any error.
2206
+ """
2207
+ try:
2208
+ import pymupdf as fitz
2209
 
2210
+ doc = fitz.open(pdf_path)
2211
+ if page_num >= len(doc):
2212
+ doc.close()
2213
+ return ""
2214
+ page = doc[page_num]
2215
+ page_text = page.get_text() or ""
2216
+ words = page.get_text("words") or []
2217
+ doc.close()
 
 
 
 
2218
 
2219
+ if not page_text or not words:
2220
  return ""
2221
 
2222
+ t = page_text.lower()
2223
+ is_nfcu = (
2224
+ ("statement of account" in t)
2225
+ and (("access no." in t) or ("routing number" in t) or ("navy federal" in t) or ("navyfederal.org" in t))
2226
  )
2227
+ if not is_nfcu:
2228
  return ""
2229
 
2230
+ # Group words into reading lines by y-position.
2231
+ buckets = []
2232
+ for w in words:
2233
+ if len(w) < 5:
2234
+ continue
2235
+ x0 = float(w[0])
2236
+ y0 = float(w[1])
2237
+ token = w[4].strip()
2238
+ if not token:
2239
+ continue
2240
+ bi = None
2241
+ for i, (yb, _) in enumerate(buckets):
2242
+ if abs(yb - y0) <= 2.0:
2243
+ bi = i
2244
+ break
2245
+ if bi is None:
2246
+ buckets.append((y0, [(x0, token)]))
2247
+ else:
2248
+ buckets[bi][1].append((x0, token))
2249
+
2250
+ lines = []
2251
+ for y, arr in sorted(buckets, key=lambda x: x[0]):
2252
+ toks = [tok for _, tok in sorted(arr, key=lambda x: x[0])]
2253
+ xs = [x for x, _ in sorted(arr, key=lambda x: x[0])]
2254
+ lines.append((y, xs, toks, " ".join(toks)))
2255
+
2256
+ date_re = re.compile(r"^\d{2}-\d{2}$")
2257
+ money_re = re.compile(r"^[\d,]+\.\d{2}-?$")
2258
+ acct_re = re.compile(r"^(Business Checking|Mbr Business Savings)\s*-\s*(\d+)")
2259
+ stop_markers = (
2260
+ "Items Paid",
2261
+ "Average Daily Balance",
2262
+ "2024 Year to Date",
2263
+ "Disclosure",
2264
+ "What to Do",
2265
+ "Errors",
2266
+ "Payments",
2267
+ "SAVINGS DIVIDENDS",
2268
+ )
2269
 
2270
+ def parse_account_rows(start_idx):
2271
+ rows = []
2272
+ i = start_idx
2273
+ pending = None
2274
+ while i < len(lines):
2275
+ _, _, toks, text = lines[i]
2276
+ if not text:
2277
+ i += 1
2278
+ continue
2279
+
2280
+ # Section/account boundaries
2281
+ if acct_re.match(text) and i != start_idx:
2282
+ break
2283
+ if any(text.startswith(s) for s in stop_markers):
2284
+ break
2285
+ if text in ("Checking", "Savings"):
2286
+ break
2287
+ if text.startswith("Date Transaction Detail"):
2288
+ i += 1
2289
+ continue
2290
 
2291
+ def flush_pending():
2292
+ nonlocal pending
2293
+ if pending is not None:
2294
+ rows.append(pending)
2295
+ pending = None
2296
+
2297
+ if toks and date_re.match(toks[0]):
2298
+ flush_pending()
2299
+ date = toks[0]
2300
+ rest = toks[1:]
2301
+ mvals = [tok for tok in rest if money_re.match(tok)]
2302
+ desc_tokens = [tok for tok in rest if not money_re.match(tok)]
2303
+
2304
+ amount = ""
2305
+ balance = ""
2306
+ if len(mvals) >= 2:
2307
+ amount = mvals[-2]
2308
+ balance = mvals[-1]
2309
+ elif len(mvals) == 1:
2310
+ # Many NFCU lines can be split; single amount token is usually balance.
2311
+ balance = mvals[-1]
2312
+
2313
+ pending = {
2314
+ "date": date,
2315
+ "desc": " ".join(desc_tokens).strip(),
2316
+ "amount": amount,
2317
+ "balance": balance,
2318
+ }
2319
+ else:
2320
+ if pending is not None:
2321
+ mvals = [tok for tok in toks if money_re.match(tok)]
2322
+ desc_tokens = [tok for tok in toks if not money_re.match(tok)]
2323
+ if desc_tokens:
2324
+ pending["desc"] = (pending["desc"] + " " + " ".join(desc_tokens)).strip()
2325
+ if len(mvals) >= 2 and (not pending["amount"] or not pending["balance"]):
2326
+ if not pending["amount"]:
2327
+ pending["amount"] = mvals[-2]
2328
+ if not pending["balance"]:
2329
+ pending["balance"] = mvals[-1]
2330
+ elif len(mvals) == 1 and not pending["balance"]:
2331
+ pending["balance"] = mvals[-1]
2332
+ # else ignore stray line
2333
+ i += 1
2334
 
2335
+ if pending is not None:
2336
+ rows.append(pending)
2337
+ # keep rows that have at least date+desc, and preferably balance
2338
+ cleaned = [r for r in rows if r["date"] and r["desc"] and (r["amount"] or r["balance"])]
2339
+ return cleaned, i
2340
 
2341
+ out = []
2342
+ i = 0
2343
+ while i < len(lines):
2344
+ _, _, _, text = lines[i]
2345
+ if not text:
2346
+ i += 1
2347
+ continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2348
 
2349
+ m = acct_re.match(text)
2350
+ if m:
2351
+ acct_name = m.group(1)
2352
+ acct_no = m.group(2)
2353
+ title = f"{acct_name} - {acct_no}"
2354
+ if i + 1 < len(lines) and "Continued" in lines[i + 1][3]:
2355
+ title += " (Continued from previous page)"
2356
+ out.append(f"<b>{title}</b>")
2357
+ out.append("<table>")
2358
+ out.append("<tr><th>Date</th><th>Transaction Detail</th><th>Amount($)</th><th>Balance($)</th></tr>")
2359
+ rows, i2 = parse_account_rows(i + 1)
2360
+ for r in rows:
2361
+ out.append(
2362
+ f"<tr><td>{r['date']}</td><td>{r['desc']}</td><td>{r['amount']}</td><td>{r['balance']}</td></tr>"
2363
+ )
2364
+ out.append("</table>")
2365
+ i = i2
2366
+ continue
2367
 
2368
+ # Keep section headers only (avoid noisy paragraph dump)
2369
+ if text in ("Checking", "Savings"):
2370
+ out.append(f"<h3>{text}</h3>")
2371
+ elif text.startswith("Items Paid"):
2372
+ out.append("<b>Items Paid</b>")
2373
+ i += 1
2374
+
2375
+ return "\n".join(out).strip()
 
 
 
 
 
2376
 
2377
  except Exception as e:
2378
+ log.warning("_parse_nfcu_page failed (page %d): %s", page_num, e)
2379
  return ""
2380
 
2381
  def normalize_html_tables(text: str) -> str:
 
2543
  try:
2544
  path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
2545
  is_pdf = path.lower().endswith(".pdf")
2546
+ is_nfcu_doc = _is_nfcu_document(path) if is_pdf else False
2547
+ nfcu_mode = is_nfcu_doc
2548
  parser = get_parser()
2549
 
2550
  page_heights = []
 
2574
 
2575
  parts = []
2576
 
2577
+ # Fix NF-1: Navy Federal full text-layer parse.
2578
+ # Must run FIRST — before header/OCR/footer — so that when NFCU is
2579
+ # detected, the entire page output is replaced with clean text-layer
2580
+ # data and none of the garbled OCR pipeline runs at all.
2581
+ # Safe no-op for all other banks (_parse_nfcu_page returns "").
2582
+ # Try NFCU structured parsing on known NFCU docs, and on page 1 as a probe.
2583
+ if is_pdf and (nfcu_mode or page_num == 0):
2584
+ nfcu_html = _parse_nfcu_page(path, page_num)
2585
+ if nfcu_html:
2586
+ nfcu_mode = True
2587
+ all_pages.append(nfcu_html)
2588
+ continue # skip ALL OCR processing for this page
2589
+ if nfcu_mode:
2590
+ log.warning("NFCU parser returned empty output for page %d", page_num + 1)
2591
+
2592
  # Header inclusion: PDF text -> band words -> OCR band
2593
  hdr = ""
2594
  if is_pdf:
 
2639
 
2640
  parts.append(stabilized)
2641
 
 
 
 
 
 
 
 
 
 
 
 
2642
  # Footer extraction: PDF text layer -> band words -> OCR crop
2643
  # Deduplication guard prevents double-printing when body OCR already captured it.
2644
  if ENABLE_FOOTER_OCR and page_num < len(page_images):