rehan953 commited on
Commit
b56c268
·
verified ·
1 Parent(s): f51e068

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +171 -318
app.py CHANGED
@@ -594,12 +594,15 @@ def _extract_fused_header_artifacts(header_row):
594
  """
595
  Generic detector for the OCR artifact where the first data row of a table
596
  gets fused into the header cells during OCR.
 
597
  The artifact pattern (BOTH signals must fire simultaneously):
598
  - Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
599
  e.g. "DESCRIPTIONBeginning Balance" "NARRATIONOpening Balance"
600
  - Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
601
  e.g. "BALANCE$3,447.10" "AMOUNT 5,000.00"
 
602
  Two-signal guard prevents false positives on clean PDFs from any bank.
 
603
  Returns:
604
  (cleaned_header : list[str], recovered_row : list[str] | None)
605
  """
@@ -704,12 +707,15 @@ def _promote_misplaced_header_row(grid):
704
  """
705
  Detects and fixes the OCR artifact where real column headers land in a
706
  <td> data row instead of the <th> header row.
 
707
  Two scenarios handled:
 
708
  Scenario A — Simple misplaced header (e.g. TD Bank):
709
  <th>ACCOUNT ACTIVITY</th> ← section title, not a real header
710
  <td>DATE DESCRIPTION</td> ... ← real column headers in a data row
711
  <td>01/05 ...</td> ← data
712
  → Promote the keyword row to header, discard the section title rows above.
 
713
  Scenario B — Metadata header + misplaced column headers (e.g. Citi page 1):
714
  <th>208479667</th> <th>Beginning Balance:$20.48 Ending Balance:$3.46</th>
715
  <td>Date Description</td> <td>Debits</td> <td>Credits</td> <td>Balance</td>
@@ -719,6 +725,7 @@ def _promote_misplaced_header_row(grid):
719
  → Preserve metadata row cells as a plain-text prefix OUTSIDE the table
720
  by embedding them as a leading data row with colspan (kept for reference).
721
  → Split any "Date Description" fused cells in data rows.
 
722
  Guard conditions (no-op if not met — safe for all other PDFs):
723
  - Current header keyword score < threshold.
724
  - A data row within the first 5 rows has keyword score >= threshold.
@@ -814,8 +821,10 @@ def _fix_fused_keyvalue_rows(grid):
814
  Fix rows where label+value are fused into the first cell with all other
815
  cells empty — common in Interest Summary / Fee Summary sections appended
816
  to a transaction table by OCR.
 
817
  e.g. "Beginning Interest Rate0.00%" → label="Beginning Interest Rate" value="0.00%"
818
  "Number of days in this Period31" → label="..." value="31"
 
819
  Guard conditions (no-op if not met — safe for all other tables):
820
  - All cells except first must be empty.
821
  - A non-space character must immediately precede the digit run (fused).
@@ -903,9 +912,11 @@ def _fix_fused_keyvalue_rows(grid):
903
  def _extract_textlayer_rows(pdf_path: str, page_num: int):
904
  """
905
  Extract structured transaction rows from the PDF text layer using pymupdf.
 
906
  Returns list of {"date": str, "desc": str, "amount": str} dicts, or []
907
  if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
908
  transaction rows are found (guards against non-transaction pages).
 
909
  Supports multiple date formats:
910
  - Numeric: MM/DD, MM/DD/YY, MM/DD/YYYY (Chase, TD, BoA, Citi)
911
  - Month-name: Jan 16, Feb 7, Mar 05 (Capital One, Amex)
@@ -1036,6 +1047,7 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
1036
  log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
1037
  return []
1038
 
 
1039
  def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1040
  """
1041
  Johnson Bank: when pdfminer column extraction fails (e.g. pymupdf line text or
@@ -1069,15 +1081,18 @@ def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1069
  out.append(t)
1070
  return out
1071
 
 
1072
  def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
1073
  """
1074
  Parse Johnson Bank Checks and Daily Account Balance from pdfminer column-separated
1075
  page text and return as HTML tables.
 
1076
  pdfminer outputs multi-column sections as separate column lists:
1077
  Date Number Amount
1078
  04-14 3259 45.72
1079
  becomes:
1080
  "Date\n04-14\n04-09\n\nNumber\n3259\n3260\n\nAmount\n45.72\n1,628.00"
 
1081
  Uses pdfminer (always available) not pymupdf.
1082
  """
1083
  if not text_layer:
@@ -1196,6 +1211,7 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
1196
  """
1197
  Compare OCR table rows against PDF text-layer rows and inject any rows
1198
  the OCR model silently dropped (typically identical consecutive rows).
 
1199
  Guard conditions (all must pass for any injection):
1200
  1. PDF text layer must exist and yield >= 2 transaction rows.
1201
  2. The OCR table must have DATE + DESCRIPTION + AMOUNT columns.
@@ -1440,13 +1456,16 @@ def _reconstruct_separator_rows(grid):
1440
  """
1441
  Detect and reconstruct mid-table separator / label rows that OCR has
1442
  incorrectly split across columns.
 
1443
  The artifact (Bank of America and similar):
1444
  A section label like "Card account # XXXX XXXX XXXX 2889" sits between
1445
  transaction rows as a full-width label. OCR splits the trailing digits
1446
  across columns because they are right-aligned, producing variations like:
 
1447
  2-cell split: ["Card account # XXXX XXXX XXXX 2", "", "889"]
1448
  3-cell split: ["Card account # XXXX XXXX XXXX", "2", "889"]
1449
  clean 1-cell: ["Card account # XXXX XXXX XXXX 2889", "", ""]
 
1450
  Detection criteria (ALL must hold — guards safe for all other PDFs):
1451
  1. First cell is NOT a date token (transaction rows always start with date).
1452
  2. First cell contains letters (it is a label, not a bare number).
@@ -1456,6 +1475,7 @@ def _reconstruct_separator_rows(grid):
1456
  — these are the split-off tails of the account/card number.
1457
  4. There must be at least one non-empty cell after the first (to detect
1458
  the split; single-cell rows are also handled as clean label rows).
 
1459
  The fix:
1460
  Concatenate all non-empty cells in order, place the result in col 0,
1461
  blank all other cells.
@@ -1531,15 +1551,19 @@ def _merge_split_rows(grid):
1531
  """
1532
  Fix OCR artifact where a transaction row is split across two <tr> rows
1533
  because the description wrapped to a second line in the source PDF.
 
1534
  Two patterns detected (Hardin County Bank and similar monospace statements):
 
1535
  Pattern A — description + empty continuation:
1536
  Row N: [desc, '', '', '', ''] ← description, no date/money
1537
  Row N+1: ['', debit, credit, date, bal] ← money/date, no description
1538
  → Merge into: [desc, debit, credit, date, bal]
 
1539
  Pattern B — description + text continuation + money:
1540
  Row N: [desc, '', '', '', ''] ← first line of description
1541
  Row N+1: [desc_cont, debit, credit, date, bal] ← continuation + money
1542
  → Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
 
1543
  Guard conditions (no-op unless both rows match the pattern):
1544
  - Row N must have non-empty col 0 (description).
1545
  - Row N must have empty date column (col 3 or wherever DATE is).
@@ -1631,9 +1655,11 @@ def _extract_fused_desc_amount(grid):
1631
  """
1632
  Fix OCR artifact where a trailing money amount gets fused into the
1633
  description cell instead of its own debit/credit column.
 
1634
  Example (Hardin County Bank):
1635
  OCR: ['CHASE CREDIT CRD EPAY 8319249882 500.00', '', '', '04/11/25', '16,699.04']
1636
  Fix: ['CHASE CREDIT CRD EPAY 8319249882', '500.00', '', '04/11/25', '16,699.04']
 
1637
  Guard conditions (all must hold):
1638
  - Description ends with a space + money amount (NNN.NN or N,NNN.NN).
1639
  - ALL debit AND credit columns are empty for this row.
@@ -1692,6 +1718,7 @@ def _is_junk_table(grid):
1692
  """
1693
  Detect and suppress known junk tables that are OCR artifacts from
1694
  page headers/footers, not real transaction data.
 
1695
  Current patterns:
1696
  1. CUSTOMER INFORMATION fragment tables (First Horizon Bank):
1697
  Single-column table with header "CUSTOMER INFORMATION" and
@@ -1714,12 +1741,15 @@ def _is_junk_table(grid):
1714
  def _split_fused_multicolumn_header(grid):
1715
  """
1716
  Fix OCR artifact where multiple column headers are fused into one <th> cell.
 
1717
  Example (First Horizon Bank alternating pages):
1718
  Fused: ['DATE DESCRIPTION CARD #', 'DEPOSIT', 'WITHDRAWAL']
1719
  Correct: ['DATE', 'DESCRIPTION', 'DEPOSIT', 'WITHDRAWAL', 'CARD #']
 
1720
  The data rows also have date + description + card# all in col 0:
1721
  Fused: ['01/16 PURCHASE - UBER TRIP... 4919', '', '$21.02']
1722
  Correct: ['01/16', 'PURCHASE - UBER TRIP...', '', '$21.02', '4919']
 
1723
  Detection: first header cell contains 2+ keyword terms separated by spaces,
1724
  AND remaining header cells are valid money column keywords.
1725
  """
@@ -1832,6 +1862,7 @@ def _normalize_daily_balance_table(grid):
1832
  - OCR produces N DATE columns followed by N BALANCE columns
1833
  instead of interleaved DATE|BALANCE|DATE|BALANCE pairs
1834
  - Multiple dates are stacked in one cell "01/02\n01/03\n01/04\n01/05"
 
1835
  Reorders columns and expands stacked date cells into proper rows.
1836
  Only fires when ALL of these hold:
1837
  - Header has exactly 2N columns where N >= 2
@@ -1909,7 +1940,9 @@ def _normalize_checks_paid_table(grid):
1909
  - Remaining headers = "AMOUNT AMOUNT AMOUNT"
1910
  - Data col0 = "01/24 4701 01/18 4704 * 01/18 916831 *" (all groups fused)
1911
  - Remaining data cells = amount values
 
1912
  Expands to proper: DATE | CHECK # | AMOUNT | DATE | CHECK # | AMOUNT | ...
 
1913
  Detection: first header cell matches pattern (DATE CHECK #)+
1914
  and remaining headers are all AMOUNT.
1915
  """
@@ -2158,335 +2191,157 @@ def convert_plaintext_bank_sections(text: str) -> str:
2158
  return "\n".join(out)
2159
 
2160
  # --------------------------
2161
- # Navy Federal full-page text-layer parser (Fix NF-1)
2162
  # --------------------------
2163
 
2164
- def _is_nfcu_document(pdf_path: str) -> bool:
2165
- """
2166
- Strict detector for Navy Federal statements.
2167
- Used to gate NF-specific parsing so other banks are unaffected.
2168
  """
2169
- try:
2170
- import pymupdf as fitz
2171
- doc = fitz.open(pdf_path)
2172
- pages_to_check = min(2, len(doc))
2173
- text = ""
2174
- for i in range(pages_to_check):
2175
- text += "\n" + (doc[i].get_text() or "")
2176
- doc.close()
2177
- t = text.lower()
2178
- return (
2179
- ("statement of account" in t)
2180
- and ("access no." in t or "routing number" in t)
2181
- and ("navy federal" in t or "navyfederal.org" in t)
2182
- )
2183
- except Exception:
2184
- return False
2185
 
 
 
2186
 
2187
- def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
2188
- """
2189
- Full text-layer parser for Navy Federal Credit Union statements.
2190
- GLM-OCR fails on NFCU pages because:
2191
- - The summary table has 5 wide columns with no visible borders
2192
- - Transaction tables have Amount($) and Balance($) spatially far right
2193
- - Multi-line descriptions (wrapped lines) confuse OCR table detection
2194
- This function reads the PDF text layer directly via pdfplumber and produces
2195
- correct, complete HTML for every section: summary table, transaction tables,
2196
- items paid, savings, disclosures.
2197
- Detection guards (all must pass — safe no-op for all other banks):
2198
- 1. Page text must contain 'Access No.' AND 'Statement of Account'
2199
- (unique to Navy Federal statement format)
2200
- 2. pdfplumber must return extractable text (not a scanned page)
2201
- Returns complete HTML string for the page, or "" if not NFCU / any error.
2202
  """
2203
  try:
2204
- text = ""
 
 
 
 
 
2205
  try:
2206
- import pdfplumber
2207
- with pdfplumber.open(pdf_path) as pdf:
2208
- if page_num >= len(pdf.pages):
2209
- return ""
2210
- page = pdf.pages[page_num]
2211
- text = page.extract_text() or ""
2212
- except Exception:
2213
- # Fallback to pymupdf text extraction when pdfplumber isn't available.
2214
- import pymupdf as fitz
2215
- doc = fitz.open(pdf_path)
2216
- if page_num >= len(doc):
2217
- doc.close()
2218
  return ""
2219
- text = doc[page_num].get_text() or ""
2220
- doc.close()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2221
 
2222
- if not text:
2223
  return ""
2224
 
2225
- # Guard: must match NFCU format on this page OR on page 0
2226
- # (pages 2+ still have "Access No." and "Statement of Account")
2227
- t = text.lower()
2228
- is_nfcu = (
2229
- ("statement of account" in t)
2230
- and (
2231
- ("access no." in t)
2232
- or ("routing number" in t)
2233
- or ("navy federal" in t)
2234
- or ("navyfederal.org" in t)
2235
- )
2236
  )
2237
- if not is_nfcu:
2238
  return ""
2239
 
2240
- lines = [l.strip() for l in text.splitlines()]
 
 
 
2241
 
2242
- # ── Regex patterns ──────────��──────────────────────────────────────
2243
- # Full transaction: "02-03 Description 7.68- 3,974.66"
2244
- TXN_FULL = re.compile(
2245
- r'^(\d{2}-\d{2})\s+(.+?)\s+([\d,]+\.\d{2}-?)\s+([\d,]+\.\d{2}-?)$'
2246
- )
2247
- # Transaction with continuation: next line has "Location amt bal"
2248
- TXN_NOAMT = re.compile(r'^(\d{2}-\d{2})\s+(.+)$')
2249
- # Continuation line ending with amount + balance
2250
- CONT_MONEY = re.compile(r'^(.+?)\s+([\d,]+\.\d{2}-?)\s+([\d,]+\.\d{2}-?)$')
2251
- # Items Paid two-column row
2252
- ITEMS_2COL = re.compile(
2253
- r'^(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})'
2254
- r'\s+(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})$'
2255
- )
2256
- ITEMS_1COL = re.compile(
2257
- r'^(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})$'
2258
- )
2259
- BOILERPLATE = re.compile(
2260
- r'^(Page \d+ of \d+|Access No\.\s*\d+|Statement Period'
2261
- r'|\d{2}/\d{2}/\d{2}\s*-\s*\d{2}/\d{2}/\d{2}'
2262
- r'|Statement of Account|For VIRTUAL EDUCATION STATION'
2263
- r'|PO Box \d+.*|navyfederal\.org.*'
2264
- r'|Say\s*"?Yes"?\s*to Paperless'
2265
- r'|Insured\s*by\s*NCUA.*|InsuredbyNCUA.*'
2266
- r'|IfyouhavenT?.*|Togetstarted.*|It\'saneasy.*'
2267
- r'|digital statements via Mobile'
2268
- r'|Navy Federal Online Banking\.)$'
2269
- )
2270
- SKIP_LINE = re.compile(
2271
- r'^(REMITTANCE RECEIVED|VIRTUAL EDUCATION STATION|18511991'
2272
- r'|DEPOSIT\s*VOUCHER|FOR MAIL USE ONLY|DEPOSITS MAY NOT'
2273
- r'|ACCOUNT\s*NUMBER|MARK\s*"?X"?\s*TO CHANGE|ADDRESS/ORDER'
2274
- r'|ITEMS ON REVERSE|NFCU|PO\s*BOX\s*3100|MERRIFIELD'
2275
- r'|405715|CHANGE OF ADDRESS|PLEASE PRINT'
2276
- r'|RANK/RATE|ADDRESS\(NO|CITY|STATE|ZIP|SIGNATURE'
2277
- r'|EFFECTIVE DATE|HOME TELEPHONE|DAYTIME TELEPHONE'
2278
- r'|- -|\( \)|Say Yes to Paperless).*$',
2279
- re.IGNORECASE
2280
- )
2281
 
2282
- # ── Helper: parse transaction block ────────────────────────────────
2283
- def parse_txn_block(i, stop_words):
2284
- rows = []
2285
- while i < len(lines):
2286
- l = lines[i]
2287
- if not l:
2288
- i += 1; continue
2289
- if BOILERPLATE.match(l) or SKIP_LINE.match(l):
2290
- i += 1; continue
2291
- if any(l == sw or l.startswith(sw) for sw in stop_words):
2292
- break
2293
 
2294
- # Full single-line transaction
2295
- m = TXN_FULL.match(l)
2296
- if m:
2297
- date, desc, amt, bal = m.group(1), m.group(2), m.group(3), m.group(4)
2298
- rows.append(f'<tr><td>{date}</td><td>{desc}</td><td>{amt}</td><td>{bal}</td></tr>')
2299
- i += 1
2300
- continue
2301
 
2302
- # Multi-line: first line has date + partial desc, next has rest+amt+bal
2303
- mn = TXN_NOAMT.match(l)
2304
- if mn:
2305
- date, desc1 = mn.group(1), mn.group(2)
2306
- i += 1
2307
- if i < len(lines):
2308
- nxt = lines[i]
2309
- mc = CONT_MONEY.match(nxt)
2310
- if mc:
2311
- desc2, amt, bal = mc.group(1), mc.group(2), mc.group(3)
2312
- full_desc = (desc1 + ' ' + desc2).strip()
2313
- rows.append(f'<tr><td>{date}</td><td>{full_desc}</td><td>{amt}</td><td>{bal}</td></tr>')
2314
- i += 1
2315
- continue
2316
- # Next line is not a continuation — treat as balance-only
2317
- rows.append(f'<tr><td>{date}</td><td>{desc1}</td><td></td><td></td></tr>')
2318
- continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2319
 
2320
- i += 1
2321
- return rows, i
2322
 
2323
- # ── Main parse loop ────────────────────────────────────────────────
2324
- out = []
2325
- i = 0
2326
- TXN_STOP = [
2327
- 'Average Daily Balance', 'Items Paid', 'Savings', 'Checking',
2328
- 'Mbr Business Savings -', '2024 Year to Date', 'Your share balance',
2329
- 'Disclosure', 'What to Do', 'Errors Related', 'Errors Within', 'Payments'
2330
  ]
2331
-
2332
- while i < len(lines):
2333
- l = lines[i]
2334
-
2335
- if not l or BOILERPLATE.match(l) or SKIP_LINE.match(l):
2336
- i += 1; continue
2337
-
2338
- # ── Summary of your deposit accounts ──────────────────────────
2339
- if l == 'Summary of your deposit accounts':
2340
- i += 1
2341
- # Skip the two header lines ("Previous Deposits/..." and "Balance Credits...")
2342
- while i < len(lines) and not re.match(r'^(Business|Mbr|Totals)', lines[i]):
2343
- i += 1
2344
-
2345
- out.append('<b>Summary of your deposit accounts</b>')
2346
- out.append('<table>')
2347
- out.append('<tr><th>Account</th><th>Previous Balance</th>'
2348
- '<th>Deposits/Credits</th><th>Withdrawals/Debits</th>'
2349
- '<th>Ending Balance</th><th>YTD Dividends</th></tr>')
2350
-
2351
- while i < len(lines):
2352
- l2 = lines[i]
2353
- if not l2 or re.match(r'^(Checking$|#|REMITTANCE)', l2):
2354
- break
2355
-
2356
- if l2.startswith('Totals '):
2357
- parts = l2.split()
2358
- vals = parts[1:]
2359
- while len(vals) < 5: vals.append('')
2360
- cells = ''.join(f'<td>{v}</td>' for v in vals[:5])
2361
- out.append(f'<tr><td><b>Totals</b></td>{cells}</tr>')
2362
- i += 1
2363
- break
2364
-
2365
- if re.match(r'^(Business Checking|Mbr Business Savings)', l2):
2366
- acct_name = l2
2367
- i += 1
2368
- if i < len(lines):
2369
- val_line = lines[i]
2370
- parts = val_line.split()
2371
- if parts and re.match(r'^\d{7,}$', parts[0]):
2372
- acct_num = parts[0]
2373
- vals = parts[1:]
2374
- else:
2375
- acct_num = ''
2376
- vals = parts
2377
- while len(vals) < 5: vals.append('')
2378
- cells = ''.join(f'<td>{v}</td>' for v in vals[:5])
2379
- name_cell = f'{acct_name} ({acct_num})' if acct_num else acct_name
2380
- out.append(f'<tr><td>{name_cell}</td>{cells}</tr>')
2381
- i += 1
2382
- continue
2383
- i += 1
2384
-
2385
- out.append('</table>')
2386
- continue
2387
-
2388
- # ── Section headers ────────────────────────────────────────────
2389
- if l in ('Checking', 'Savings'):
2390
- out.append(f'<h3>{l}</h3>')
2391
- i += 1
2392
- continue
2393
-
2394
- # ── Account transaction table ──────────────────────────────────
2395
- m_acct = re.match(
2396
- r'^(Business Checking|Mbr Business Savings) - (\d+)(.*)?$', l
2397
- )
2398
- if m_acct:
2399
- acct_line = l
2400
- i += 1
2401
- if i < len(lines) and 'Continued' in lines[i]:
2402
- acct_line += ' ' + lines[i]
2403
- i += 1
2404
- if i < len(lines) and lines[i].startswith('Date '):
2405
- i += 1 # skip "Date Transaction Detail Amount($) Balance($)"
2406
-
2407
- out.append(f'<b>{acct_line}</b>')
2408
- out.append('<table>')
2409
- out.append('<tr><th>Date</th><th>Transaction Detail</th>'
2410
- '<th>Amount($)</th><th>Balance($)</th></tr>')
2411
-
2412
- rows, i = parse_txn_block(i, TXN_STOP)
2413
- out.extend(rows)
2414
- out.append('</table>')
2415
- continue
2416
-
2417
- # ── Items Paid ─────────────────────────────────────────────────
2418
- if l == 'Items Paid' or l.startswith('Items Paid (Continued'):
2419
- out.append(f'<b>{l}</b>')
2420
- out.append('<table>')
2421
- out.append('<tr><th>Date</th><th>Item</th><th>Amount($)</th>'
2422
- '<th>Date</th><th>Item</th><th>Amount($)</th></tr>')
2423
- i += 1
2424
- if i < len(lines) and lines[i].startswith('Date '):
2425
- i += 1
2426
-
2427
- ITEMS_STOP = ['Savings', '2024 Year to Date', 'Checking',
2428
- 'Disclosure', 'What to Do']
2429
- while i < len(lines):
2430
- l2 = lines[i]
2431
- if not l2: i += 1; continue
2432
- if any(l2 == sw or l2.startswith(sw) for sw in ITEMS_STOP):
2433
- break
2434
- if BOILERPLATE.match(l2) or SKIP_LINE.match(l2):
2435
- i += 1; continue
2436
-
2437
- m2c = ITEMS_2COL.match(l2)
2438
- if m2c:
2439
- d1,t1,a1,d2,t2,a2 = m2c.groups()
2440
- out.append(f'<tr><td>{d1}</td><td>{t1}</td><td>{a1}</td>'
2441
- f'<td>{d2}</td><td>{t2}</td><td>{a2}</td></tr>')
2442
- i += 1; continue
2443
-
2444
- m1c = ITEMS_1COL.match(l2)
2445
- if m1c:
2446
- d1,t1,a1 = m1c.groups()
2447
- out.append(f'<tr><td>{d1}</td><td>{t1}</td><td>{a1}</td>'
2448
- f'<td></td><td></td><td></td></tr>')
2449
- i += 1; continue
2450
- i += 1
2451
-
2452
- out.append('</table>')
2453
- continue
2454
-
2455
- # ── Average Daily Balance ──────────────────────────────────────
2456
- if l.startswith('Average Daily Balance'):
2457
- out.append(f'<p>{l}</p>')
2458
- i += 1; continue
2459
-
2460
- # ── 2024 Year to Date ──────────────────────────────────────────
2461
- if l.startswith('2024 Year to Date'):
2462
- out.append(f'<p><b>{l}</b></p>')
2463
- i += 1
2464
- while i < len(lines) and lines[i] and not BOILERPLATE.match(lines[i]):
2465
- out.append(f'<p>{lines[i]}</p>')
2466
- i += 1
2467
- continue
2468
-
2469
- # ── Your share balance is overdrawn ───────────────────────────
2470
- if l.startswith('Your share balance'):
2471
- out.append(f'<p><i>{l}</i></p>')
2472
- i += 1; continue
2473
-
2474
- # ── Disclosure page — output as plain text ────────────────────
2475
- if any(l.startswith(x) for x in
2476
- ('Disclosure', 'WhattoDoif', 'What to Do', 'Errors', 'Payments',
2477
- 'ErrorsRelated', 'ErrorsWithin')):
2478
- out.append(f'<p>{l}</p>')
2479
- i += 1; continue
2480
-
2481
- # Default: keep non-empty lines
2482
- if l:
2483
- out.append(f'<p>{l}</p>')
2484
- i += 1
2485
-
2486
- return '\n'.join(out)
2487
 
2488
  except Exception as e:
2489
- log.warning("_parse_nfcu_page failed (page %d): %s", page_num, e)
2490
  return ""
2491
 
2492
  def normalize_html_tables(text: str) -> str:
@@ -2654,7 +2509,6 @@ def run_ocr(uploaded_file):
2654
  try:
2655
  path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
2656
  is_pdf = path.lower().endswith(".pdf")
2657
- is_nfcu_doc = _is_nfcu_document(path) if is_pdf else False
2658
  parser = get_parser()
2659
 
2660
  page_heights = []
@@ -2684,18 +2538,6 @@ def run_ocr(uploaded_file):
2684
 
2685
  parts = []
2686
 
2687
- # Fix NF-1: Navy Federal full text-layer parse.
2688
- # Must run FIRST — before header/OCR/footer — so that when NFCU is
2689
- # detected, the entire page output is replaced with clean text-layer
2690
- # data and none of the garbled OCR pipeline runs at all.
2691
- # Safe no-op for all other banks (_parse_nfcu_page returns "").
2692
- if is_pdf and is_nfcu_doc:
2693
- nfcu_html = _parse_nfcu_page(path, page_num)
2694
- if nfcu_html:
2695
- all_pages.append(nfcu_html)
2696
- continue # skip ALL OCR processing for this page
2697
- log.warning("NFCU parser returned empty output for page %d", page_num + 1)
2698
-
2699
  # Header inclusion: PDF text -> band words -> OCR band
2700
  hdr = ""
2701
  if is_pdf:
@@ -2746,6 +2588,17 @@ def run_ocr(uploaded_file):
2746
 
2747
  parts.append(stabilized)
2748
 
 
 
 
 
 
 
 
 
 
 
 
2749
  # Footer extraction: PDF text layer -> band words -> OCR crop
2750
  # Deduplication guard prevents double-printing when body OCR already captured it.
2751
  if ENABLE_FOOTER_OCR and page_num < len(page_images):
 
594
  """
595
  Generic detector for the OCR artifact where the first data row of a table
596
  gets fused into the header cells during OCR.
597
+
598
  The artifact pattern (BOTH signals must fire simultaneously):
599
  - Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
600
  e.g. "DESCRIPTIONBeginning Balance" "NARRATIONOpening Balance"
601
  - Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
602
  e.g. "BALANCE$3,447.10" "AMOUNT 5,000.00"
603
+
604
  Two-signal guard prevents false positives on clean PDFs from any bank.
605
+
606
  Returns:
607
  (cleaned_header : list[str], recovered_row : list[str] | None)
608
  """
 
707
  """
708
  Detects and fixes the OCR artifact where real column headers land in a
709
  <td> data row instead of the <th> header row.
710
+
711
  Two scenarios handled:
712
+
713
  Scenario A — Simple misplaced header (e.g. TD Bank):
714
  <th>ACCOUNT ACTIVITY</th> ← section title, not a real header
715
  <td>DATE DESCRIPTION</td> ... ← real column headers in a data row
716
  <td>01/05 ...</td> ← data
717
  → Promote the keyword row to header, discard the section title rows above.
718
+
719
  Scenario B — Metadata header + misplaced column headers (e.g. Citi page 1):
720
  <th>208479667</th> <th>Beginning Balance:$20.48 Ending Balance:$3.46</th>
721
  <td>Date Description</td> <td>Debits</td> <td>Credits</td> <td>Balance</td>
 
725
  → Preserve metadata row cells as a plain-text prefix OUTSIDE the table
726
  by embedding them as a leading data row with colspan (kept for reference).
727
  → Split any "Date Description" fused cells in data rows.
728
+
729
  Guard conditions (no-op if not met — safe for all other PDFs):
730
  - Current header keyword score < threshold.
731
  - A data row within the first 5 rows has keyword score >= threshold.
 
821
  Fix rows where label+value are fused into the first cell with all other
822
  cells empty — common in Interest Summary / Fee Summary sections appended
823
  to a transaction table by OCR.
824
+
825
  e.g. "Beginning Interest Rate0.00%" → label="Beginning Interest Rate" value="0.00%"
826
  "Number of days in this Period31" → label="..." value="31"
827
+
828
  Guard conditions (no-op if not met — safe for all other tables):
829
  - All cells except first must be empty.
830
  - A non-space character must immediately precede the digit run (fused).
 
912
  def _extract_textlayer_rows(pdf_path: str, page_num: int):
913
  """
914
  Extract structured transaction rows from the PDF text layer using pymupdf.
915
+
916
  Returns list of {"date": str, "desc": str, "amount": str} dicts, or []
917
  if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
918
  transaction rows are found (guards against non-transaction pages).
919
+
920
  Supports multiple date formats:
921
  - Numeric: MM/DD, MM/DD/YY, MM/DD/YYYY (Chase, TD, BoA, Citi)
922
  - Month-name: Jan 16, Feb 7, Mar 05 (Capital One, Amex)
 
1047
  log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
1048
  return []
1049
 
1050
+
1051
  def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
1052
  """
1053
  Johnson Bank: when pdfminer column extraction fails (e.g. pymupdf line text or
 
1081
  out.append(t)
1082
  return out
1083
 
1084
+
1085
  def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
1086
  """
1087
  Parse Johnson Bank Checks and Daily Account Balance from pdfminer column-separated
1088
  page text and return as HTML tables.
1089
+
1090
  pdfminer outputs multi-column sections as separate column lists:
1091
  Date Number Amount
1092
  04-14 3259 45.72
1093
  becomes:
1094
  "Date\n04-14\n04-09\n\nNumber\n3259\n3260\n\nAmount\n45.72\n1,628.00"
1095
+
1096
  Uses pdfminer (always available) not pymupdf.
1097
  """
1098
  if not text_layer:
 
1211
  """
1212
  Compare OCR table rows against PDF text-layer rows and inject any rows
1213
  the OCR model silently dropped (typically identical consecutive rows).
1214
+
1215
  Guard conditions (all must pass for any injection):
1216
  1. PDF text layer must exist and yield >= 2 transaction rows.
1217
  2. The OCR table must have DATE + DESCRIPTION + AMOUNT columns.
 
1456
  """
1457
  Detect and reconstruct mid-table separator / label rows that OCR has
1458
  incorrectly split across columns.
1459
+
1460
  The artifact (Bank of America and similar):
1461
  A section label like "Card account # XXXX XXXX XXXX 2889" sits between
1462
  transaction rows as a full-width label. OCR splits the trailing digits
1463
  across columns because they are right-aligned, producing variations like:
1464
+
1465
  2-cell split: ["Card account # XXXX XXXX XXXX 2", "", "889"]
1466
  3-cell split: ["Card account # XXXX XXXX XXXX", "2", "889"]
1467
  clean 1-cell: ["Card account # XXXX XXXX XXXX 2889", "", ""]
1468
+
1469
  Detection criteria (ALL must hold — guards safe for all other PDFs):
1470
  1. First cell is NOT a date token (transaction rows always start with date).
1471
  2. First cell contains letters (it is a label, not a bare number).
 
1475
  — these are the split-off tails of the account/card number.
1476
  4. There must be at least one non-empty cell after the first (to detect
1477
  the split; single-cell rows are also handled as clean label rows).
1478
+
1479
  The fix:
1480
  Concatenate all non-empty cells in order, place the result in col 0,
1481
  blank all other cells.
 
1551
  """
1552
  Fix OCR artifact where a transaction row is split across two <tr> rows
1553
  because the description wrapped to a second line in the source PDF.
1554
+
1555
  Two patterns detected (Hardin County Bank and similar monospace statements):
1556
+
1557
  Pattern A — description + empty continuation:
1558
  Row N: [desc, '', '', '', ''] ← description, no date/money
1559
  Row N+1: ['', debit, credit, date, bal] ← money/date, no description
1560
  → Merge into: [desc, debit, credit, date, bal]
1561
+
1562
  Pattern B — description + text continuation + money:
1563
  Row N: [desc, '', '', '', ''] ← first line of description
1564
  Row N+1: [desc_cont, debit, credit, date, bal] ← continuation + money
1565
  → Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
1566
+
1567
  Guard conditions (no-op unless both rows match the pattern):
1568
  - Row N must have non-empty col 0 (description).
1569
  - Row N must have empty date column (col 3 or wherever DATE is).
 
1655
  """
1656
  Fix OCR artifact where a trailing money amount gets fused into the
1657
  description cell instead of its own debit/credit column.
1658
+
1659
  Example (Hardin County Bank):
1660
  OCR: ['CHASE CREDIT CRD EPAY 8319249882 500.00', '', '', '04/11/25', '16,699.04']
1661
  Fix: ['CHASE CREDIT CRD EPAY 8319249882', '500.00', '', '04/11/25', '16,699.04']
1662
+
1663
  Guard conditions (all must hold):
1664
  - Description ends with a space + money amount (NNN.NN or N,NNN.NN).
1665
  - ALL debit AND credit columns are empty for this row.
 
1718
  """
1719
  Detect and suppress known junk tables that are OCR artifacts from
1720
  page headers/footers, not real transaction data.
1721
+
1722
  Current patterns:
1723
  1. CUSTOMER INFORMATION fragment tables (First Horizon Bank):
1724
  Single-column table with header "CUSTOMER INFORMATION" and
 
1741
  def _split_fused_multicolumn_header(grid):
1742
  """
1743
  Fix OCR artifact where multiple column headers are fused into one <th> cell.
1744
+
1745
  Example (First Horizon Bank alternating pages):
1746
  Fused: ['DATE DESCRIPTION CARD #', 'DEPOSIT', 'WITHDRAWAL']
1747
  Correct: ['DATE', 'DESCRIPTION', 'DEPOSIT', 'WITHDRAWAL', 'CARD #']
1748
+
1749
  The data rows also have date + description + card# all in col 0:
1750
  Fused: ['01/16 PURCHASE - UBER TRIP... 4919', '', '$21.02']
1751
  Correct: ['01/16', 'PURCHASE - UBER TRIP...', '', '$21.02', '4919']
1752
+
1753
  Detection: first header cell contains 2+ keyword terms separated by spaces,
1754
  AND remaining header cells are valid money column keywords.
1755
  """
 
1862
  - OCR produces N DATE columns followed by N BALANCE columns
1863
  instead of interleaved DATE|BALANCE|DATE|BALANCE pairs
1864
  - Multiple dates are stacked in one cell "01/02\n01/03\n01/04\n01/05"
1865
+
1866
  Reorders columns and expands stacked date cells into proper rows.
1867
  Only fires when ALL of these hold:
1868
  - Header has exactly 2N columns where N >= 2
 
1940
  - Remaining headers = "AMOUNT AMOUNT AMOUNT"
1941
  - Data col0 = "01/24 4701 01/18 4704 * 01/18 916831 *" (all groups fused)
1942
  - Remaining data cells = amount values
1943
+
1944
  Expands to proper: DATE | CHECK # | AMOUNT | DATE | CHECK # | AMOUNT | ...
1945
+
1946
  Detection: first header cell matches pattern (DATE CHECK #)+
1947
  and remaining headers are all AMOUNT.
1948
  """
 
2191
  return "\n".join(out)
2192
 
2193
  # --------------------------
2194
+ # Navy Federal Summary Table extraction (Fix NF-1)
2195
  # --------------------------
2196
 
2197
+ def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
 
 
 
2198
  """
2199
+ Extract Navy Federal "Summary of your deposit accounts" table from the PDF
2200
+ text layer using word coordinates.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2201
 
2202
+ GLM-OCR fails on this table because the five data columns are spread very
2203
+ wide across the page with no visible borders.
2204
 
2205
+ Uses word x/y coordinates to reconstruct the table spatially.
2206
+ Tries pymupdf first (HuggingFace), falls back to pdfplumber (local).
2207
+
2208
+ Detection guards (safe no-op for all other banks):
2209
+ 1. Word "Summary" in top 55% of page.
2210
+ 2. Words "deposit" or "accounts" within 15pt of it.
2211
+
2212
+ Returns HTML table string, or "" if not found/applicable.
 
 
 
 
 
 
 
2213
  """
2214
  try:
2215
+ from collections import defaultdict as _dd
2216
+
2217
+ words = None # list of (x0, top, text)
2218
+ page_h = 1000
2219
+
2220
+ # Try pymupdf first (HuggingFace environment)
2221
  try:
2222
+ import pymupdf as _fitz_s
2223
+ _doc_s = _fitz_s.open(pdf_path)
2224
+ if page_num >= len(_doc_s):
2225
+ _doc_s.close()
 
 
 
 
 
 
 
 
2226
  return ""
2227
+ _pg_s = _doc_s[page_num]
2228
+ page_h = _pg_s.rect.height
2229
+ words = [(w[0], w[1], w[4]) for w in _pg_s.get_text("words")]
2230
+ _doc_s.close()
2231
+ except Exception:
2232
+ pass
2233
+
2234
+ # Fallback: pdfplumber (local environment)
2235
+ if not words:
2236
+ try:
2237
+ import pdfplumber as _plumber_s
2238
+ with _plumber_s.open(pdf_path) as _pdf_s:
2239
+ if page_num >= len(_pdf_s.pages):
2240
+ return ""
2241
+ page_h = _pdf_s.pages[page_num].height
2242
+ words = [(w["x0"], w["top"], w["text"])
2243
+ for w in _pdf_s.pages[page_num].extract_words()]
2244
+ except Exception:
2245
+ pass
2246
 
2247
+ if not words:
2248
  return ""
2249
 
2250
+ # Guard 1: "Summary" word in top 55% of page
2251
+ summary_word = next(
2252
+ (w for w in words if w[2].lower() == "summary" and w[1] < page_h * 0.55),
2253
+ None
 
 
 
 
 
 
 
2254
  )
2255
+ if not summary_word:
2256
  return ""
2257
 
2258
+ # Guard 2: "deposit" or "accounts" within 15pt
2259
+ nearby = [w[2].lower() for w in words if abs(w[1] - summary_word[1]) < 15]
2260
+ if "deposit" not in nearby and "accounts" not in nearby:
2261
+ return ""
2262
 
2263
+ # Band: 15-115pt below the heading
2264
+ y_start = summary_word[1] + 15
2265
+ y_end = summary_word[1] + 115
2266
+ band = [w for w in words if y_start <= w[1] <= y_end]
2267
+ if not band:
2268
+ return ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2269
 
2270
+ # Group into lines by y-bucket (5pt tolerance)
2271
+ lines_by_y = _dd(list)
2272
+ for w in band:
2273
+ lines_by_y[round(w[1] / 5) * 5].append(w)
2274
+ rows_raw = [sorted(lines_by_y[y], key=lambda w: w[0])
2275
+ for y in sorted(lines_by_y)]
 
 
 
 
 
2276
 
2277
+ if len(rows_raw) < 3:
2278
+ return ""
 
 
 
 
 
2279
 
2280
+ # Column centers from first two header rows
2281
+ header_xs = sorted(w[0] for r in rows_raw[:2] for w in r)
2282
+ col_centers = []
2283
+ cluster = [header_xs[0]]
2284
+ for x in header_xs[1:]:
2285
+ if x - cluster[-1] > 30:
2286
+ col_centers.append(sum(cluster) / len(cluster))
2287
+ cluster = [x]
2288
+ else:
2289
+ cluster.append(x)
2290
+ col_centers.append(sum(cluster) / len(cluster))
2291
+
2292
+ def _assign(x):
2293
+ return min(range(len(col_centers)), key=lambda i: abs(col_centers[i] - x))
2294
+
2295
+ # Build column labels from first two rows
2296
+ col_labels = [""] * len(col_centers)
2297
+ for r in rows_raw[:2]:
2298
+ for w in r:
2299
+ ci = _assign(w[0])
2300
+ col_labels[ci] = (col_labels[ci] + " " + w[2]).strip()
2301
+
2302
+ # Parse data rows
2303
+ _MONEY = re.compile(r"^\$?-?[\d,]+\.\d{2}-?$")
2304
+ merged, cur_name, cur_vals = [], [], {}
2305
+
2306
+ for line in rows_raw[2:]:
2307
+ texts = [w[2] for w in line]
2308
+ has_money = any(_MONEY.match(t) for t in texts)
2309
+ is_totals = any(w[2].lower() == "totals" for w in line)
2310
+
2311
+ if has_money:
2312
+ for w in line:
2313
+ if not re.match(r"^\d{7,}$", w[2]) and w[2].lower() != "totals":
2314
+ cur_vals[_assign(w[0])] = w[2]
2315
+ acct_num = next((w[2] for w in line if re.match(r"^\d{7,}$", w[2])), "")
2316
+ name = " ".join(cur_name)
2317
+ if acct_num:
2318
+ name = (name + " " + acct_num).strip()
2319
+ if is_totals:
2320
+ name = "Totals"
2321
+ merged.append((name, dict(cur_vals)))
2322
+ cur_name, cur_vals = [], {}
2323
+ else:
2324
+ cur_name.extend(texts)
2325
 
2326
+ if not merged:
2327
+ return ""
2328
 
2329
+ n = len(col_labels)
2330
+ hdr = "<tr><th>Account</th>" + "".join(f"<th>{l}</th>" for l in col_labels) + "</tr>"
2331
+ rows_html = [
2332
+ "<tr><td>" + name + "</td>" +
2333
+ "".join(f"<td>{vals.get(i, '')}</td>" for i in range(n)) +
2334
+ "</tr>"
2335
+ for name, vals in merged
2336
  ]
2337
+ return (
2338
+ "Summary of your deposit accounts\n"
2339
+ "<table>\n" + hdr + "\n" +
2340
+ "\n".join(rows_html) + "\n</table>"
2341
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2342
 
2343
  except Exception as e:
2344
+ log.warning("_extract_nfcu_summary_table failed (page %d): %s", page_num, e)
2345
  return ""
2346
 
2347
  def normalize_html_tables(text: str) -> str:
 
2509
  try:
2510
  path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
2511
  is_pdf = path.lower().endswith(".pdf")
 
2512
  parser = get_parser()
2513
 
2514
  page_heights = []
 
2538
 
2539
  parts = []
2540
 
 
 
 
 
 
 
 
 
 
 
 
 
2541
  # Header inclusion: PDF text -> band words -> OCR band
2542
  hdr = ""
2543
  if is_pdf:
 
2588
 
2589
  parts.append(stabilized)
2590
 
2591
+ # Fix NF-1: prepend Navy Federal summary table from text layer.
2592
+ # Replaces the garbled OCR summary table with correct data.
2593
+ # Safe no-op for all non-NFCU PDFs (returns "" when guards fail).
2594
+ if is_pdf:
2595
+ try:
2596
+ nfcu_summary = _extract_nfcu_summary_table(path, page_num)
2597
+ if nfcu_summary:
2598
+ parts.insert(0, nfcu_summary)
2599
+ except Exception:
2600
+ pass
2601
+
2602
  # Footer extraction: PDF text layer -> band words -> OCR crop
2603
  # Deduplication guard prevents double-printing when body OCR already captured it.
2604
  if ENABLE_FOOTER_OCR and page_num < len(page_images):