Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -594,12 +594,15 @@ def _extract_fused_header_artifacts(header_row):
|
|
| 594 |
"""
|
| 595 |
Generic detector for the OCR artifact where the first data row of a table
|
| 596 |
gets fused into the header cells during OCR.
|
|
|
|
| 597 |
The artifact pattern (BOTH signals must fire simultaneously):
|
| 598 |
- Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
|
| 599 |
e.g. "DESCRIPTIONBeginning Balance" "NARRATIONOpening Balance"
|
| 600 |
- Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
|
| 601 |
e.g. "BALANCE$3,447.10" "AMOUNT 5,000.00"
|
|
|
|
| 602 |
Two-signal guard prevents false positives on clean PDFs from any bank.
|
|
|
|
| 603 |
Returns:
|
| 604 |
(cleaned_header : list[str], recovered_row : list[str] | None)
|
| 605 |
"""
|
|
@@ -704,12 +707,15 @@ def _promote_misplaced_header_row(grid):
|
|
| 704 |
"""
|
| 705 |
Detects and fixes the OCR artifact where real column headers land in a
|
| 706 |
<td> data row instead of the <th> header row.
|
|
|
|
| 707 |
Two scenarios handled:
|
|
|
|
| 708 |
Scenario A — Simple misplaced header (e.g. TD Bank):
|
| 709 |
<th>ACCOUNT ACTIVITY</th> ← section title, not a real header
|
| 710 |
<td>DATE DESCRIPTION</td> ... ← real column headers in a data row
|
| 711 |
<td>01/05 ...</td> ← data
|
| 712 |
→ Promote the keyword row to header, discard the section title rows above.
|
|
|
|
| 713 |
Scenario B — Metadata header + misplaced column headers (e.g. Citi page 1):
|
| 714 |
<th>208479667</th> <th>Beginning Balance:$20.48 Ending Balance:$3.46</th>
|
| 715 |
<td>Date Description</td> <td>Debits</td> <td>Credits</td> <td>Balance</td>
|
|
@@ -719,6 +725,7 @@ def _promote_misplaced_header_row(grid):
|
|
| 719 |
→ Preserve metadata row cells as a plain-text prefix OUTSIDE the table
|
| 720 |
by embedding them as a leading data row with colspan (kept for reference).
|
| 721 |
→ Split any "Date Description" fused cells in data rows.
|
|
|
|
| 722 |
Guard conditions (no-op if not met — safe for all other PDFs):
|
| 723 |
- Current header keyword score < threshold.
|
| 724 |
- A data row within the first 5 rows has keyword score >= threshold.
|
|
@@ -814,8 +821,10 @@ def _fix_fused_keyvalue_rows(grid):
|
|
| 814 |
Fix rows where label+value are fused into the first cell with all other
|
| 815 |
cells empty — common in Interest Summary / Fee Summary sections appended
|
| 816 |
to a transaction table by OCR.
|
|
|
|
| 817 |
e.g. "Beginning Interest Rate0.00%" → label="Beginning Interest Rate" value="0.00%"
|
| 818 |
"Number of days in this Period31" → label="..." value="31"
|
|
|
|
| 819 |
Guard conditions (no-op if not met — safe for all other tables):
|
| 820 |
- All cells except first must be empty.
|
| 821 |
- A non-space character must immediately precede the digit run (fused).
|
|
@@ -903,9 +912,11 @@ def _fix_fused_keyvalue_rows(grid):
|
|
| 903 |
def _extract_textlayer_rows(pdf_path: str, page_num: int):
|
| 904 |
"""
|
| 905 |
Extract structured transaction rows from the PDF text layer using pymupdf.
|
|
|
|
| 906 |
Returns list of {"date": str, "desc": str, "amount": str} dicts, or []
|
| 907 |
if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
|
| 908 |
transaction rows are found (guards against non-transaction pages).
|
|
|
|
| 909 |
Supports multiple date formats:
|
| 910 |
- Numeric: MM/DD, MM/DD/YY, MM/DD/YYYY (Chase, TD, BoA, Citi)
|
| 911 |
- Month-name: Jan 16, Feb 7, Mar 05 (Capital One, Amex)
|
|
@@ -1036,6 +1047,7 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
|
|
| 1036 |
log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
|
| 1037 |
return []
|
| 1038 |
|
|
|
|
| 1039 |
def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
|
| 1040 |
"""
|
| 1041 |
Johnson Bank: when pdfminer column extraction fails (e.g. pymupdf line text or
|
|
@@ -1069,15 +1081,18 @@ def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
|
|
| 1069 |
out.append(t)
|
| 1070 |
return out
|
| 1071 |
|
|
|
|
| 1072 |
def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
|
| 1073 |
"""
|
| 1074 |
Parse Johnson Bank Checks and Daily Account Balance from pdfminer column-separated
|
| 1075 |
page text and return as HTML tables.
|
|
|
|
| 1076 |
pdfminer outputs multi-column sections as separate column lists:
|
| 1077 |
Date Number Amount
|
| 1078 |
04-14 3259 45.72
|
| 1079 |
becomes:
|
| 1080 |
"Date\n04-14\n04-09\n\nNumber\n3259\n3260\n\nAmount\n45.72\n1,628.00"
|
|
|
|
| 1081 |
Uses pdfminer (always available) not pymupdf.
|
| 1082 |
"""
|
| 1083 |
if not text_layer:
|
|
@@ -1196,6 +1211,7 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
|
|
| 1196 |
"""
|
| 1197 |
Compare OCR table rows against PDF text-layer rows and inject any rows
|
| 1198 |
the OCR model silently dropped (typically identical consecutive rows).
|
|
|
|
| 1199 |
Guard conditions (all must pass for any injection):
|
| 1200 |
1. PDF text layer must exist and yield >= 2 transaction rows.
|
| 1201 |
2. The OCR table must have DATE + DESCRIPTION + AMOUNT columns.
|
|
@@ -1440,13 +1456,16 @@ def _reconstruct_separator_rows(grid):
|
|
| 1440 |
"""
|
| 1441 |
Detect and reconstruct mid-table separator / label rows that OCR has
|
| 1442 |
incorrectly split across columns.
|
|
|
|
| 1443 |
The artifact (Bank of America and similar):
|
| 1444 |
A section label like "Card account # XXXX XXXX XXXX 2889" sits between
|
| 1445 |
transaction rows as a full-width label. OCR splits the trailing digits
|
| 1446 |
across columns because they are right-aligned, producing variations like:
|
|
|
|
| 1447 |
2-cell split: ["Card account # XXXX XXXX XXXX 2", "", "889"]
|
| 1448 |
3-cell split: ["Card account # XXXX XXXX XXXX", "2", "889"]
|
| 1449 |
clean 1-cell: ["Card account # XXXX XXXX XXXX 2889", "", ""]
|
|
|
|
| 1450 |
Detection criteria (ALL must hold — guards safe for all other PDFs):
|
| 1451 |
1. First cell is NOT a date token (transaction rows always start with date).
|
| 1452 |
2. First cell contains letters (it is a label, not a bare number).
|
|
@@ -1456,6 +1475,7 @@ def _reconstruct_separator_rows(grid):
|
|
| 1456 |
— these are the split-off tails of the account/card number.
|
| 1457 |
4. There must be at least one non-empty cell after the first (to detect
|
| 1458 |
the split; single-cell rows are also handled as clean label rows).
|
|
|
|
| 1459 |
The fix:
|
| 1460 |
Concatenate all non-empty cells in order, place the result in col 0,
|
| 1461 |
blank all other cells.
|
|
@@ -1531,15 +1551,19 @@ def _merge_split_rows(grid):
|
|
| 1531 |
"""
|
| 1532 |
Fix OCR artifact where a transaction row is split across two <tr> rows
|
| 1533 |
because the description wrapped to a second line in the source PDF.
|
|
|
|
| 1534 |
Two patterns detected (Hardin County Bank and similar monospace statements):
|
|
|
|
| 1535 |
Pattern A — description + empty continuation:
|
| 1536 |
Row N: [desc, '', '', '', ''] ← description, no date/money
|
| 1537 |
Row N+1: ['', debit, credit, date, bal] ← money/date, no description
|
| 1538 |
→ Merge into: [desc, debit, credit, date, bal]
|
|
|
|
| 1539 |
Pattern B — description + text continuation + money:
|
| 1540 |
Row N: [desc, '', '', '', ''] ← first line of description
|
| 1541 |
Row N+1: [desc_cont, debit, credit, date, bal] ← continuation + money
|
| 1542 |
→ Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
|
|
|
|
| 1543 |
Guard conditions (no-op unless both rows match the pattern):
|
| 1544 |
- Row N must have non-empty col 0 (description).
|
| 1545 |
- Row N must have empty date column (col 3 or wherever DATE is).
|
|
@@ -1631,9 +1655,11 @@ def _extract_fused_desc_amount(grid):
|
|
| 1631 |
"""
|
| 1632 |
Fix OCR artifact where a trailing money amount gets fused into the
|
| 1633 |
description cell instead of its own debit/credit column.
|
|
|
|
| 1634 |
Example (Hardin County Bank):
|
| 1635 |
OCR: ['CHASE CREDIT CRD EPAY 8319249882 500.00', '', '', '04/11/25', '16,699.04']
|
| 1636 |
Fix: ['CHASE CREDIT CRD EPAY 8319249882', '500.00', '', '04/11/25', '16,699.04']
|
|
|
|
| 1637 |
Guard conditions (all must hold):
|
| 1638 |
- Description ends with a space + money amount (NNN.NN or N,NNN.NN).
|
| 1639 |
- ALL debit AND credit columns are empty for this row.
|
|
@@ -1692,6 +1718,7 @@ def _is_junk_table(grid):
|
|
| 1692 |
"""
|
| 1693 |
Detect and suppress known junk tables that are OCR artifacts from
|
| 1694 |
page headers/footers, not real transaction data.
|
|
|
|
| 1695 |
Current patterns:
|
| 1696 |
1. CUSTOMER INFORMATION fragment tables (First Horizon Bank):
|
| 1697 |
Single-column table with header "CUSTOMER INFORMATION" and
|
|
@@ -1714,12 +1741,15 @@ def _is_junk_table(grid):
|
|
| 1714 |
def _split_fused_multicolumn_header(grid):
|
| 1715 |
"""
|
| 1716 |
Fix OCR artifact where multiple column headers are fused into one <th> cell.
|
|
|
|
| 1717 |
Example (First Horizon Bank alternating pages):
|
| 1718 |
Fused: ['DATE DESCRIPTION CARD #', 'DEPOSIT', 'WITHDRAWAL']
|
| 1719 |
Correct: ['DATE', 'DESCRIPTION', 'DEPOSIT', 'WITHDRAWAL', 'CARD #']
|
|
|
|
| 1720 |
The data rows also have date + description + card# all in col 0:
|
| 1721 |
Fused: ['01/16 PURCHASE - UBER TRIP... 4919', '', '$21.02']
|
| 1722 |
Correct: ['01/16', 'PURCHASE - UBER TRIP...', '', '$21.02', '4919']
|
|
|
|
| 1723 |
Detection: first header cell contains 2+ keyword terms separated by spaces,
|
| 1724 |
AND remaining header cells are valid money column keywords.
|
| 1725 |
"""
|
|
@@ -1832,6 +1862,7 @@ def _normalize_daily_balance_table(grid):
|
|
| 1832 |
- OCR produces N DATE columns followed by N BALANCE columns
|
| 1833 |
instead of interleaved DATE|BALANCE|DATE|BALANCE pairs
|
| 1834 |
- Multiple dates are stacked in one cell "01/02\n01/03\n01/04\n01/05"
|
|
|
|
| 1835 |
Reorders columns and expands stacked date cells into proper rows.
|
| 1836 |
Only fires when ALL of these hold:
|
| 1837 |
- Header has exactly 2N columns where N >= 2
|
|
@@ -1909,7 +1940,9 @@ def _normalize_checks_paid_table(grid):
|
|
| 1909 |
- Remaining headers = "AMOUNT AMOUNT AMOUNT"
|
| 1910 |
- Data col0 = "01/24 4701 01/18 4704 * 01/18 916831 *" (all groups fused)
|
| 1911 |
- Remaining data cells = amount values
|
|
|
|
| 1912 |
Expands to proper: DATE | CHECK # | AMOUNT | DATE | CHECK # | AMOUNT | ...
|
|
|
|
| 1913 |
Detection: first header cell matches pattern (DATE CHECK #)+
|
| 1914 |
and remaining headers are all AMOUNT.
|
| 1915 |
"""
|
|
@@ -2158,335 +2191,157 @@ def convert_plaintext_bank_sections(text: str) -> str:
|
|
| 2158 |
return "\n".join(out)
|
| 2159 |
|
| 2160 |
# --------------------------
|
| 2161 |
-
# Navy Federal
|
| 2162 |
# --------------------------
|
| 2163 |
|
| 2164 |
-
def
|
| 2165 |
-
"""
|
| 2166 |
-
Strict detector for Navy Federal statements.
|
| 2167 |
-
Used to gate NF-specific parsing so other banks are unaffected.
|
| 2168 |
"""
|
| 2169 |
-
|
| 2170 |
-
|
| 2171 |
-
doc = fitz.open(pdf_path)
|
| 2172 |
-
pages_to_check = min(2, len(doc))
|
| 2173 |
-
text = ""
|
| 2174 |
-
for i in range(pages_to_check):
|
| 2175 |
-
text += "\n" + (doc[i].get_text() or "")
|
| 2176 |
-
doc.close()
|
| 2177 |
-
t = text.lower()
|
| 2178 |
-
return (
|
| 2179 |
-
("statement of account" in t)
|
| 2180 |
-
and ("access no." in t or "routing number" in t)
|
| 2181 |
-
and ("navy federal" in t or "navyfederal.org" in t)
|
| 2182 |
-
)
|
| 2183 |
-
except Exception:
|
| 2184 |
-
return False
|
| 2185 |
|
|
|
|
|
|
|
| 2186 |
|
| 2187 |
-
|
| 2188 |
-
|
| 2189 |
-
|
| 2190 |
-
|
| 2191 |
-
|
| 2192 |
-
|
| 2193 |
-
|
| 2194 |
-
|
| 2195 |
-
correct, complete HTML for every section: summary table, transaction tables,
|
| 2196 |
-
items paid, savings, disclosures.
|
| 2197 |
-
Detection guards (all must pass — safe no-op for all other banks):
|
| 2198 |
-
1. Page text must contain 'Access No.' AND 'Statement of Account'
|
| 2199 |
-
(unique to Navy Federal statement format)
|
| 2200 |
-
2. pdfplumber must return extractable text (not a scanned page)
|
| 2201 |
-
Returns complete HTML string for the page, or "" if not NFCU / any error.
|
| 2202 |
"""
|
| 2203 |
try:
|
| 2204 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2205 |
try:
|
| 2206 |
-
import
|
| 2207 |
-
|
| 2208 |
-
|
| 2209 |
-
|
| 2210 |
-
page = pdf.pages[page_num]
|
| 2211 |
-
text = page.extract_text() or ""
|
| 2212 |
-
except Exception:
|
| 2213 |
-
# Fallback to pymupdf text extraction when pdfplumber isn't available.
|
| 2214 |
-
import pymupdf as fitz
|
| 2215 |
-
doc = fitz.open(pdf_path)
|
| 2216 |
-
if page_num >= len(doc):
|
| 2217 |
-
doc.close()
|
| 2218 |
return ""
|
| 2219 |
-
|
| 2220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2221 |
|
| 2222 |
-
if not
|
| 2223 |
return ""
|
| 2224 |
|
| 2225 |
-
# Guard:
|
| 2226 |
-
|
| 2227 |
-
|
| 2228 |
-
|
| 2229 |
-
("statement of account" in t)
|
| 2230 |
-
and (
|
| 2231 |
-
("access no." in t)
|
| 2232 |
-
or ("routing number" in t)
|
| 2233 |
-
or ("navy federal" in t)
|
| 2234 |
-
or ("navyfederal.org" in t)
|
| 2235 |
-
)
|
| 2236 |
)
|
| 2237 |
-
if not
|
| 2238 |
return ""
|
| 2239 |
|
| 2240 |
-
|
|
|
|
|
|
|
|
|
|
| 2241 |
|
| 2242 |
-
#
|
| 2243 |
-
|
| 2244 |
-
|
| 2245 |
-
|
| 2246 |
-
|
| 2247 |
-
|
| 2248 |
-
TXN_NOAMT = re.compile(r'^(\d{2}-\d{2})\s+(.+)$')
|
| 2249 |
-
# Continuation line ending with amount + balance
|
| 2250 |
-
CONT_MONEY = re.compile(r'^(.+?)\s+([\d,]+\.\d{2}-?)\s+([\d,]+\.\d{2}-?)$')
|
| 2251 |
-
# Items Paid two-column row
|
| 2252 |
-
ITEMS_2COL = re.compile(
|
| 2253 |
-
r'^(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})'
|
| 2254 |
-
r'\s+(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})$'
|
| 2255 |
-
)
|
| 2256 |
-
ITEMS_1COL = re.compile(
|
| 2257 |
-
r'^(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})$'
|
| 2258 |
-
)
|
| 2259 |
-
BOILERPLATE = re.compile(
|
| 2260 |
-
r'^(Page \d+ of \d+|Access No\.\s*\d+|Statement Period'
|
| 2261 |
-
r'|\d{2}/\d{2}/\d{2}\s*-\s*\d{2}/\d{2}/\d{2}'
|
| 2262 |
-
r'|Statement of Account|For VIRTUAL EDUCATION STATION'
|
| 2263 |
-
r'|PO Box \d+.*|navyfederal\.org.*'
|
| 2264 |
-
r'|Say\s*"?Yes"?\s*to Paperless'
|
| 2265 |
-
r'|Insured\s*by\s*NCUA.*|InsuredbyNCUA.*'
|
| 2266 |
-
r'|IfyouhavenT?.*|Togetstarted.*|It\'saneasy.*'
|
| 2267 |
-
r'|digital statements via Mobile'
|
| 2268 |
-
r'|Navy Federal Online Banking\.)$'
|
| 2269 |
-
)
|
| 2270 |
-
SKIP_LINE = re.compile(
|
| 2271 |
-
r'^(REMITTANCE RECEIVED|VIRTUAL EDUCATION STATION|18511991'
|
| 2272 |
-
r'|DEPOSIT\s*VOUCHER|FOR MAIL USE ONLY|DEPOSITS MAY NOT'
|
| 2273 |
-
r'|ACCOUNT\s*NUMBER|MARK\s*"?X"?\s*TO CHANGE|ADDRESS/ORDER'
|
| 2274 |
-
r'|ITEMS ON REVERSE|NFCU|PO\s*BOX\s*3100|MERRIFIELD'
|
| 2275 |
-
r'|405715|CHANGE OF ADDRESS|PLEASE PRINT'
|
| 2276 |
-
r'|RANK/RATE|ADDRESS\(NO|CITY|STATE|ZIP|SIGNATURE'
|
| 2277 |
-
r'|EFFECTIVE DATE|HOME TELEPHONE|DAYTIME TELEPHONE'
|
| 2278 |
-
r'|- -|\( \)|Say Yes to Paperless).*$',
|
| 2279 |
-
re.IGNORECASE
|
| 2280 |
-
)
|
| 2281 |
|
| 2282 |
-
#
|
| 2283 |
-
|
| 2284 |
-
|
| 2285 |
-
|
| 2286 |
-
|
| 2287 |
-
|
| 2288 |
-
i += 1; continue
|
| 2289 |
-
if BOILERPLATE.match(l) or SKIP_LINE.match(l):
|
| 2290 |
-
i += 1; continue
|
| 2291 |
-
if any(l == sw or l.startswith(sw) for sw in stop_words):
|
| 2292 |
-
break
|
| 2293 |
|
| 2294 |
-
|
| 2295 |
-
|
| 2296 |
-
if m:
|
| 2297 |
-
date, desc, amt, bal = m.group(1), m.group(2), m.group(3), m.group(4)
|
| 2298 |
-
rows.append(f'<tr><td>{date}</td><td>{desc}</td><td>{amt}</td><td>{bal}</td></tr>')
|
| 2299 |
-
i += 1
|
| 2300 |
-
continue
|
| 2301 |
|
| 2302 |
-
|
| 2303 |
-
|
| 2304 |
-
|
| 2305 |
-
|
| 2306 |
-
|
| 2307 |
-
|
| 2308 |
-
|
| 2309 |
-
|
| 2310 |
-
|
| 2311 |
-
|
| 2312 |
-
|
| 2313 |
-
|
| 2314 |
-
|
| 2315 |
-
|
| 2316 |
-
|
| 2317 |
-
|
| 2318 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2319 |
|
| 2320 |
-
|
| 2321 |
-
return
|
| 2322 |
|
| 2323 |
-
|
| 2324 |
-
|
| 2325 |
-
|
| 2326 |
-
|
| 2327 |
-
|
| 2328 |
-
|
| 2329 |
-
|
| 2330 |
]
|
| 2331 |
-
|
| 2332 |
-
|
| 2333 |
-
|
| 2334 |
-
|
| 2335 |
-
|
| 2336 |
-
i += 1; continue
|
| 2337 |
-
|
| 2338 |
-
# ── Summary of your deposit accounts ──────────────────────────
|
| 2339 |
-
if l == 'Summary of your deposit accounts':
|
| 2340 |
-
i += 1
|
| 2341 |
-
# Skip the two header lines ("Previous Deposits/..." and "Balance Credits...")
|
| 2342 |
-
while i < len(lines) and not re.match(r'^(Business|Mbr|Totals)', lines[i]):
|
| 2343 |
-
i += 1
|
| 2344 |
-
|
| 2345 |
-
out.append('<b>Summary of your deposit accounts</b>')
|
| 2346 |
-
out.append('<table>')
|
| 2347 |
-
out.append('<tr><th>Account</th><th>Previous Balance</th>'
|
| 2348 |
-
'<th>Deposits/Credits</th><th>Withdrawals/Debits</th>'
|
| 2349 |
-
'<th>Ending Balance</th><th>YTD Dividends</th></tr>')
|
| 2350 |
-
|
| 2351 |
-
while i < len(lines):
|
| 2352 |
-
l2 = lines[i]
|
| 2353 |
-
if not l2 or re.match(r'^(Checking$|#|REMITTANCE)', l2):
|
| 2354 |
-
break
|
| 2355 |
-
|
| 2356 |
-
if l2.startswith('Totals '):
|
| 2357 |
-
parts = l2.split()
|
| 2358 |
-
vals = parts[1:]
|
| 2359 |
-
while len(vals) < 5: vals.append('')
|
| 2360 |
-
cells = ''.join(f'<td>{v}</td>' for v in vals[:5])
|
| 2361 |
-
out.append(f'<tr><td><b>Totals</b></td>{cells}</tr>')
|
| 2362 |
-
i += 1
|
| 2363 |
-
break
|
| 2364 |
-
|
| 2365 |
-
if re.match(r'^(Business Checking|Mbr Business Savings)', l2):
|
| 2366 |
-
acct_name = l2
|
| 2367 |
-
i += 1
|
| 2368 |
-
if i < len(lines):
|
| 2369 |
-
val_line = lines[i]
|
| 2370 |
-
parts = val_line.split()
|
| 2371 |
-
if parts and re.match(r'^\d{7,}$', parts[0]):
|
| 2372 |
-
acct_num = parts[0]
|
| 2373 |
-
vals = parts[1:]
|
| 2374 |
-
else:
|
| 2375 |
-
acct_num = ''
|
| 2376 |
-
vals = parts
|
| 2377 |
-
while len(vals) < 5: vals.append('')
|
| 2378 |
-
cells = ''.join(f'<td>{v}</td>' for v in vals[:5])
|
| 2379 |
-
name_cell = f'{acct_name} ({acct_num})' if acct_num else acct_name
|
| 2380 |
-
out.append(f'<tr><td>{name_cell}</td>{cells}</tr>')
|
| 2381 |
-
i += 1
|
| 2382 |
-
continue
|
| 2383 |
-
i += 1
|
| 2384 |
-
|
| 2385 |
-
out.append('</table>')
|
| 2386 |
-
continue
|
| 2387 |
-
|
| 2388 |
-
# ── Section headers ────────────────────────────────────────────
|
| 2389 |
-
if l in ('Checking', 'Savings'):
|
| 2390 |
-
out.append(f'<h3>{l}</h3>')
|
| 2391 |
-
i += 1
|
| 2392 |
-
continue
|
| 2393 |
-
|
| 2394 |
-
# ── Account transaction table ──────────────────────────────────
|
| 2395 |
-
m_acct = re.match(
|
| 2396 |
-
r'^(Business Checking|Mbr Business Savings) - (\d+)(.*)?$', l
|
| 2397 |
-
)
|
| 2398 |
-
if m_acct:
|
| 2399 |
-
acct_line = l
|
| 2400 |
-
i += 1
|
| 2401 |
-
if i < len(lines) and 'Continued' in lines[i]:
|
| 2402 |
-
acct_line += ' ' + lines[i]
|
| 2403 |
-
i += 1
|
| 2404 |
-
if i < len(lines) and lines[i].startswith('Date '):
|
| 2405 |
-
i += 1 # skip "Date Transaction Detail Amount($) Balance($)"
|
| 2406 |
-
|
| 2407 |
-
out.append(f'<b>{acct_line}</b>')
|
| 2408 |
-
out.append('<table>')
|
| 2409 |
-
out.append('<tr><th>Date</th><th>Transaction Detail</th>'
|
| 2410 |
-
'<th>Amount($)</th><th>Balance($)</th></tr>')
|
| 2411 |
-
|
| 2412 |
-
rows, i = parse_txn_block(i, TXN_STOP)
|
| 2413 |
-
out.extend(rows)
|
| 2414 |
-
out.append('</table>')
|
| 2415 |
-
continue
|
| 2416 |
-
|
| 2417 |
-
# ── Items Paid ─────────────────────────────────────────────────
|
| 2418 |
-
if l == 'Items Paid' or l.startswith('Items Paid (Continued'):
|
| 2419 |
-
out.append(f'<b>{l}</b>')
|
| 2420 |
-
out.append('<table>')
|
| 2421 |
-
out.append('<tr><th>Date</th><th>Item</th><th>Amount($)</th>'
|
| 2422 |
-
'<th>Date</th><th>Item</th><th>Amount($)</th></tr>')
|
| 2423 |
-
i += 1
|
| 2424 |
-
if i < len(lines) and lines[i].startswith('Date '):
|
| 2425 |
-
i += 1
|
| 2426 |
-
|
| 2427 |
-
ITEMS_STOP = ['Savings', '2024 Year to Date', 'Checking',
|
| 2428 |
-
'Disclosure', 'What to Do']
|
| 2429 |
-
while i < len(lines):
|
| 2430 |
-
l2 = lines[i]
|
| 2431 |
-
if not l2: i += 1; continue
|
| 2432 |
-
if any(l2 == sw or l2.startswith(sw) for sw in ITEMS_STOP):
|
| 2433 |
-
break
|
| 2434 |
-
if BOILERPLATE.match(l2) or SKIP_LINE.match(l2):
|
| 2435 |
-
i += 1; continue
|
| 2436 |
-
|
| 2437 |
-
m2c = ITEMS_2COL.match(l2)
|
| 2438 |
-
if m2c:
|
| 2439 |
-
d1,t1,a1,d2,t2,a2 = m2c.groups()
|
| 2440 |
-
out.append(f'<tr><td>{d1}</td><td>{t1}</td><td>{a1}</td>'
|
| 2441 |
-
f'<td>{d2}</td><td>{t2}</td><td>{a2}</td></tr>')
|
| 2442 |
-
i += 1; continue
|
| 2443 |
-
|
| 2444 |
-
m1c = ITEMS_1COL.match(l2)
|
| 2445 |
-
if m1c:
|
| 2446 |
-
d1,t1,a1 = m1c.groups()
|
| 2447 |
-
out.append(f'<tr><td>{d1}</td><td>{t1}</td><td>{a1}</td>'
|
| 2448 |
-
f'<td></td><td></td><td></td></tr>')
|
| 2449 |
-
i += 1; continue
|
| 2450 |
-
i += 1
|
| 2451 |
-
|
| 2452 |
-
out.append('</table>')
|
| 2453 |
-
continue
|
| 2454 |
-
|
| 2455 |
-
# ── Average Daily Balance ──────────────────────────────────────
|
| 2456 |
-
if l.startswith('Average Daily Balance'):
|
| 2457 |
-
out.append(f'<p>{l}</p>')
|
| 2458 |
-
i += 1; continue
|
| 2459 |
-
|
| 2460 |
-
# ── 2024 Year to Date ──────────────────────────────────────────
|
| 2461 |
-
if l.startswith('2024 Year to Date'):
|
| 2462 |
-
out.append(f'<p><b>{l}</b></p>')
|
| 2463 |
-
i += 1
|
| 2464 |
-
while i < len(lines) and lines[i] and not BOILERPLATE.match(lines[i]):
|
| 2465 |
-
out.append(f'<p>{lines[i]}</p>')
|
| 2466 |
-
i += 1
|
| 2467 |
-
continue
|
| 2468 |
-
|
| 2469 |
-
# ── Your share balance is overdrawn ───────────────────────────
|
| 2470 |
-
if l.startswith('Your share balance'):
|
| 2471 |
-
out.append(f'<p><i>{l}</i></p>')
|
| 2472 |
-
i += 1; continue
|
| 2473 |
-
|
| 2474 |
-
# ── Disclosure page — output as plain text ────────────────────
|
| 2475 |
-
if any(l.startswith(x) for x in
|
| 2476 |
-
('Disclosure', 'WhattoDoif', 'What to Do', 'Errors', 'Payments',
|
| 2477 |
-
'ErrorsRelated', 'ErrorsWithin')):
|
| 2478 |
-
out.append(f'<p>{l}</p>')
|
| 2479 |
-
i += 1; continue
|
| 2480 |
-
|
| 2481 |
-
# Default: keep non-empty lines
|
| 2482 |
-
if l:
|
| 2483 |
-
out.append(f'<p>{l}</p>')
|
| 2484 |
-
i += 1
|
| 2485 |
-
|
| 2486 |
-
return '\n'.join(out)
|
| 2487 |
|
| 2488 |
except Exception as e:
|
| 2489 |
-
log.warning("
|
| 2490 |
return ""
|
| 2491 |
|
| 2492 |
def normalize_html_tables(text: str) -> str:
|
|
@@ -2654,7 +2509,6 @@ def run_ocr(uploaded_file):
|
|
| 2654 |
try:
|
| 2655 |
path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
|
| 2656 |
is_pdf = path.lower().endswith(".pdf")
|
| 2657 |
-
is_nfcu_doc = _is_nfcu_document(path) if is_pdf else False
|
| 2658 |
parser = get_parser()
|
| 2659 |
|
| 2660 |
page_heights = []
|
|
@@ -2684,18 +2538,6 @@ def run_ocr(uploaded_file):
|
|
| 2684 |
|
| 2685 |
parts = []
|
| 2686 |
|
| 2687 |
-
# Fix NF-1: Navy Federal full text-layer parse.
|
| 2688 |
-
# Must run FIRST — before header/OCR/footer — so that when NFCU is
|
| 2689 |
-
# detected, the entire page output is replaced with clean text-layer
|
| 2690 |
-
# data and none of the garbled OCR pipeline runs at all.
|
| 2691 |
-
# Safe no-op for all other banks (_parse_nfcu_page returns "").
|
| 2692 |
-
if is_pdf and is_nfcu_doc:
|
| 2693 |
-
nfcu_html = _parse_nfcu_page(path, page_num)
|
| 2694 |
-
if nfcu_html:
|
| 2695 |
-
all_pages.append(nfcu_html)
|
| 2696 |
-
continue # skip ALL OCR processing for this page
|
| 2697 |
-
log.warning("NFCU parser returned empty output for page %d", page_num + 1)
|
| 2698 |
-
|
| 2699 |
# Header inclusion: PDF text -> band words -> OCR band
|
| 2700 |
hdr = ""
|
| 2701 |
if is_pdf:
|
|
@@ -2746,6 +2588,17 @@ def run_ocr(uploaded_file):
|
|
| 2746 |
|
| 2747 |
parts.append(stabilized)
|
| 2748 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2749 |
# Footer extraction: PDF text layer -> band words -> OCR crop
|
| 2750 |
# Deduplication guard prevents double-printing when body OCR already captured it.
|
| 2751 |
if ENABLE_FOOTER_OCR and page_num < len(page_images):
|
|
|
|
| 594 |
"""
|
| 595 |
Generic detector for the OCR artifact where the first data row of a table
|
| 596 |
gets fused into the header cells during OCR.
|
| 597 |
+
|
| 598 |
The artifact pattern (BOTH signals must fire simultaneously):
|
| 599 |
- Signal 1 — text-fused: a header cell contains KEYWORD + free descriptive text
|
| 600 |
e.g. "DESCRIPTIONBeginning Balance" "NARRATIONOpening Balance"
|
| 601 |
- Signal 2 — money-fused: a DIFFERENT header cell contains KEYWORD + money amount
|
| 602 |
e.g. "BALANCE$3,447.10" "AMOUNT 5,000.00"
|
| 603 |
+
|
| 604 |
Two-signal guard prevents false positives on clean PDFs from any bank.
|
| 605 |
+
|
| 606 |
Returns:
|
| 607 |
(cleaned_header : list[str], recovered_row : list[str] | None)
|
| 608 |
"""
|
|
|
|
| 707 |
"""
|
| 708 |
Detects and fixes the OCR artifact where real column headers land in a
|
| 709 |
<td> data row instead of the <th> header row.
|
| 710 |
+
|
| 711 |
Two scenarios handled:
|
| 712 |
+
|
| 713 |
Scenario A — Simple misplaced header (e.g. TD Bank):
|
| 714 |
<th>ACCOUNT ACTIVITY</th> ← section title, not a real header
|
| 715 |
<td>DATE DESCRIPTION</td> ... ← real column headers in a data row
|
| 716 |
<td>01/05 ...</td> ← data
|
| 717 |
→ Promote the keyword row to header, discard the section title rows above.
|
| 718 |
+
|
| 719 |
Scenario B — Metadata header + misplaced column headers (e.g. Citi page 1):
|
| 720 |
<th>208479667</th> <th>Beginning Balance:$20.48 Ending Balance:$3.46</th>
|
| 721 |
<td>Date Description</td> <td>Debits</td> <td>Credits</td> <td>Balance</td>
|
|
|
|
| 725 |
→ Preserve metadata row cells as a plain-text prefix OUTSIDE the table
|
| 726 |
by embedding them as a leading data row with colspan (kept for reference).
|
| 727 |
→ Split any "Date Description" fused cells in data rows.
|
| 728 |
+
|
| 729 |
Guard conditions (no-op if not met — safe for all other PDFs):
|
| 730 |
- Current header keyword score < threshold.
|
| 731 |
- A data row within the first 5 rows has keyword score >= threshold.
|
|
|
|
| 821 |
Fix rows where label+value are fused into the first cell with all other
|
| 822 |
cells empty — common in Interest Summary / Fee Summary sections appended
|
| 823 |
to a transaction table by OCR.
|
| 824 |
+
|
| 825 |
e.g. "Beginning Interest Rate0.00%" → label="Beginning Interest Rate" value="0.00%"
|
| 826 |
"Number of days in this Period31" → label="..." value="31"
|
| 827 |
+
|
| 828 |
Guard conditions (no-op if not met — safe for all other tables):
|
| 829 |
- All cells except first must be empty.
|
| 830 |
- A non-space character must immediately precede the digit run (fused).
|
|
|
|
| 912 |
def _extract_textlayer_rows(pdf_path: str, page_num: int):
|
| 913 |
"""
|
| 914 |
Extract structured transaction rows from the PDF text layer using pymupdf.
|
| 915 |
+
|
| 916 |
Returns list of {"date": str, "desc": str, "amount": str} dicts, or []
|
| 917 |
if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
|
| 918 |
transaction rows are found (guards against non-transaction pages).
|
| 919 |
+
|
| 920 |
Supports multiple date formats:
|
| 921 |
- Numeric: MM/DD, MM/DD/YY, MM/DD/YYYY (Chase, TD, BoA, Citi)
|
| 922 |
- Month-name: Jan 16, Feb 7, Mar 05 (Capital One, Amex)
|
|
|
|
| 1047 |
log.debug("_extract_textlayer_rows failed (page %d): %s", page_num, e)
|
| 1048 |
return []
|
| 1049 |
|
| 1050 |
+
|
| 1051 |
def _jb_extract_checks_fallback(text_layer: str) -> List[Tuple[str, str, str]]:
|
| 1052 |
"""
|
| 1053 |
Johnson Bank: when pdfminer column extraction fails (e.g. pymupdf line text or
|
|
|
|
| 1081 |
out.append(t)
|
| 1082 |
return out
|
| 1083 |
|
| 1084 |
+
|
| 1085 |
def _extract_jb_summary_sections(text_layer: str, needs_checks: bool = True, needs_dab: bool = True) -> str:
|
| 1086 |
"""
|
| 1087 |
Parse Johnson Bank Checks and Daily Account Balance from pdfminer column-separated
|
| 1088 |
page text and return as HTML tables.
|
| 1089 |
+
|
| 1090 |
pdfminer outputs multi-column sections as separate column lists:
|
| 1091 |
Date Number Amount
|
| 1092 |
04-14 3259 45.72
|
| 1093 |
becomes:
|
| 1094 |
"Date\n04-14\n04-09\n\nNumber\n3259\n3260\n\nAmount\n45.72\n1,628.00"
|
| 1095 |
+
|
| 1096 |
Uses pdfminer (always available) not pymupdf.
|
| 1097 |
"""
|
| 1098 |
if not text_layer:
|
|
|
|
| 1211 |
"""
|
| 1212 |
Compare OCR table rows against PDF text-layer rows and inject any rows
|
| 1213 |
the OCR model silently dropped (typically identical consecutive rows).
|
| 1214 |
+
|
| 1215 |
Guard conditions (all must pass for any injection):
|
| 1216 |
1. PDF text layer must exist and yield >= 2 transaction rows.
|
| 1217 |
2. The OCR table must have DATE + DESCRIPTION + AMOUNT columns.
|
|
|
|
| 1456 |
"""
|
| 1457 |
Detect and reconstruct mid-table separator / label rows that OCR has
|
| 1458 |
incorrectly split across columns.
|
| 1459 |
+
|
| 1460 |
The artifact (Bank of America and similar):
|
| 1461 |
A section label like "Card account # XXXX XXXX XXXX 2889" sits between
|
| 1462 |
transaction rows as a full-width label. OCR splits the trailing digits
|
| 1463 |
across columns because they are right-aligned, producing variations like:
|
| 1464 |
+
|
| 1465 |
2-cell split: ["Card account # XXXX XXXX XXXX 2", "", "889"]
|
| 1466 |
3-cell split: ["Card account # XXXX XXXX XXXX", "2", "889"]
|
| 1467 |
clean 1-cell: ["Card account # XXXX XXXX XXXX 2889", "", ""]
|
| 1468 |
+
|
| 1469 |
Detection criteria (ALL must hold — guards safe for all other PDFs):
|
| 1470 |
1. First cell is NOT a date token (transaction rows always start with date).
|
| 1471 |
2. First cell contains letters (it is a label, not a bare number).
|
|
|
|
| 1475 |
— these are the split-off tails of the account/card number.
|
| 1476 |
4. There must be at least one non-empty cell after the first (to detect
|
| 1477 |
the split; single-cell rows are also handled as clean label rows).
|
| 1478 |
+
|
| 1479 |
The fix:
|
| 1480 |
Concatenate all non-empty cells in order, place the result in col 0,
|
| 1481 |
blank all other cells.
|
|
|
|
| 1551 |
"""
|
| 1552 |
Fix OCR artifact where a transaction row is split across two <tr> rows
|
| 1553 |
because the description wrapped to a second line in the source PDF.
|
| 1554 |
+
|
| 1555 |
Two patterns detected (Hardin County Bank and similar monospace statements):
|
| 1556 |
+
|
| 1557 |
Pattern A — description + empty continuation:
|
| 1558 |
Row N: [desc, '', '', '', ''] ← description, no date/money
|
| 1559 |
Row N+1: ['', debit, credit, date, bal] ← money/date, no description
|
| 1560 |
→ Merge into: [desc, debit, credit, date, bal]
|
| 1561 |
+
|
| 1562 |
Pattern B — description + text continuation + money:
|
| 1563 |
Row N: [desc, '', '', '', ''] ← first line of description
|
| 1564 |
Row N+1: [desc_cont, debit, credit, date, bal] ← continuation + money
|
| 1565 |
→ Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
|
| 1566 |
+
|
| 1567 |
Guard conditions (no-op unless both rows match the pattern):
|
| 1568 |
- Row N must have non-empty col 0 (description).
|
| 1569 |
- Row N must have empty date column (col 3 or wherever DATE is).
|
|
|
|
| 1655 |
"""
|
| 1656 |
Fix OCR artifact where a trailing money amount gets fused into the
|
| 1657 |
description cell instead of its own debit/credit column.
|
| 1658 |
+
|
| 1659 |
Example (Hardin County Bank):
|
| 1660 |
OCR: ['CHASE CREDIT CRD EPAY 8319249882 500.00', '', '', '04/11/25', '16,699.04']
|
| 1661 |
Fix: ['CHASE CREDIT CRD EPAY 8319249882', '500.00', '', '04/11/25', '16,699.04']
|
| 1662 |
+
|
| 1663 |
Guard conditions (all must hold):
|
| 1664 |
- Description ends with a space + money amount (NNN.NN or N,NNN.NN).
|
| 1665 |
- ALL debit AND credit columns are empty for this row.
|
|
|
|
| 1718 |
"""
|
| 1719 |
Detect and suppress known junk tables that are OCR artifacts from
|
| 1720 |
page headers/footers, not real transaction data.
|
| 1721 |
+
|
| 1722 |
Current patterns:
|
| 1723 |
1. CUSTOMER INFORMATION fragment tables (First Horizon Bank):
|
| 1724 |
Single-column table with header "CUSTOMER INFORMATION" and
|
|
|
|
| 1741 |
def _split_fused_multicolumn_header(grid):
|
| 1742 |
"""
|
| 1743 |
Fix OCR artifact where multiple column headers are fused into one <th> cell.
|
| 1744 |
+
|
| 1745 |
Example (First Horizon Bank alternating pages):
|
| 1746 |
Fused: ['DATE DESCRIPTION CARD #', 'DEPOSIT', 'WITHDRAWAL']
|
| 1747 |
Correct: ['DATE', 'DESCRIPTION', 'DEPOSIT', 'WITHDRAWAL', 'CARD #']
|
| 1748 |
+
|
| 1749 |
The data rows also have date + description + card# all in col 0:
|
| 1750 |
Fused: ['01/16 PURCHASE - UBER TRIP... 4919', '', '$21.02']
|
| 1751 |
Correct: ['01/16', 'PURCHASE - UBER TRIP...', '', '$21.02', '4919']
|
| 1752 |
+
|
| 1753 |
Detection: first header cell contains 2+ keyword terms separated by spaces,
|
| 1754 |
AND remaining header cells are valid money column keywords.
|
| 1755 |
"""
|
|
|
|
| 1862 |
- OCR produces N DATE columns followed by N BALANCE columns
|
| 1863 |
instead of interleaved DATE|BALANCE|DATE|BALANCE pairs
|
| 1864 |
- Multiple dates are stacked in one cell "01/02\n01/03\n01/04\n01/05"
|
| 1865 |
+
|
| 1866 |
Reorders columns and expands stacked date cells into proper rows.
|
| 1867 |
Only fires when ALL of these hold:
|
| 1868 |
- Header has exactly 2N columns where N >= 2
|
|
|
|
| 1940 |
- Remaining headers = "AMOUNT AMOUNT AMOUNT"
|
| 1941 |
- Data col0 = "01/24 4701 01/18 4704 * 01/18 916831 *" (all groups fused)
|
| 1942 |
- Remaining data cells = amount values
|
| 1943 |
+
|
| 1944 |
Expands to proper: DATE | CHECK # | AMOUNT | DATE | CHECK # | AMOUNT | ...
|
| 1945 |
+
|
| 1946 |
Detection: first header cell matches pattern (DATE CHECK #)+
|
| 1947 |
and remaining headers are all AMOUNT.
|
| 1948 |
"""
|
|
|
|
| 2191 |
return "\n".join(out)
|
| 2192 |
|
| 2193 |
# --------------------------
|
| 2194 |
+
# Navy Federal Summary Table extraction (Fix NF-1)
|
| 2195 |
# --------------------------
|
| 2196 |
|
| 2197 |
+
def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
|
|
|
|
|
|
|
|
|
|
| 2198 |
"""
|
| 2199 |
+
Extract Navy Federal "Summary of your deposit accounts" table from the PDF
|
| 2200 |
+
text layer using word coordinates.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2201 |
|
| 2202 |
+
GLM-OCR fails on this table because the five data columns are spread very
|
| 2203 |
+
wide across the page with no visible borders.
|
| 2204 |
|
| 2205 |
+
Uses word x/y coordinates to reconstruct the table spatially.
|
| 2206 |
+
Tries pymupdf first (HuggingFace), falls back to pdfplumber (local).
|
| 2207 |
+
|
| 2208 |
+
Detection guards (safe no-op for all other banks):
|
| 2209 |
+
1. Word "Summary" in top 55% of page.
|
| 2210 |
+
2. Words "deposit" or "accounts" within 15pt of it.
|
| 2211 |
+
|
| 2212 |
+
Returns HTML table string, or "" if not found/applicable.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2213 |
"""
|
| 2214 |
try:
|
| 2215 |
+
from collections import defaultdict as _dd
|
| 2216 |
+
|
| 2217 |
+
words = None # list of (x0, top, text)
|
| 2218 |
+
page_h = 1000
|
| 2219 |
+
|
| 2220 |
+
# Try pymupdf first (HuggingFace environment)
|
| 2221 |
try:
|
| 2222 |
+
import pymupdf as _fitz_s
|
| 2223 |
+
_doc_s = _fitz_s.open(pdf_path)
|
| 2224 |
+
if page_num >= len(_doc_s):
|
| 2225 |
+
_doc_s.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2226 |
return ""
|
| 2227 |
+
_pg_s = _doc_s[page_num]
|
| 2228 |
+
page_h = _pg_s.rect.height
|
| 2229 |
+
words = [(w[0], w[1], w[4]) for w in _pg_s.get_text("words")]
|
| 2230 |
+
_doc_s.close()
|
| 2231 |
+
except Exception:
|
| 2232 |
+
pass
|
| 2233 |
+
|
| 2234 |
+
# Fallback: pdfplumber (local environment)
|
| 2235 |
+
if not words:
|
| 2236 |
+
try:
|
| 2237 |
+
import pdfplumber as _plumber_s
|
| 2238 |
+
with _plumber_s.open(pdf_path) as _pdf_s:
|
| 2239 |
+
if page_num >= len(_pdf_s.pages):
|
| 2240 |
+
return ""
|
| 2241 |
+
page_h = _pdf_s.pages[page_num].height
|
| 2242 |
+
words = [(w["x0"], w["top"], w["text"])
|
| 2243 |
+
for w in _pdf_s.pages[page_num].extract_words()]
|
| 2244 |
+
except Exception:
|
| 2245 |
+
pass
|
| 2246 |
|
| 2247 |
+
if not words:
|
| 2248 |
return ""
|
| 2249 |
|
| 2250 |
+
# Guard 1: "Summary" word in top 55% of page
|
| 2251 |
+
summary_word = next(
|
| 2252 |
+
(w for w in words if w[2].lower() == "summary" and w[1] < page_h * 0.55),
|
| 2253 |
+
None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2254 |
)
|
| 2255 |
+
if not summary_word:
|
| 2256 |
return ""
|
| 2257 |
|
| 2258 |
+
# Guard 2: "deposit" or "accounts" within 15pt
|
| 2259 |
+
nearby = [w[2].lower() for w in words if abs(w[1] - summary_word[1]) < 15]
|
| 2260 |
+
if "deposit" not in nearby and "accounts" not in nearby:
|
| 2261 |
+
return ""
|
| 2262 |
|
| 2263 |
+
# Band: 15-115pt below the heading
|
| 2264 |
+
y_start = summary_word[1] + 15
|
| 2265 |
+
y_end = summary_word[1] + 115
|
| 2266 |
+
band = [w for w in words if y_start <= w[1] <= y_end]
|
| 2267 |
+
if not band:
|
| 2268 |
+
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2269 |
|
| 2270 |
+
# Group into lines by y-bucket (5pt tolerance)
|
| 2271 |
+
lines_by_y = _dd(list)
|
| 2272 |
+
for w in band:
|
| 2273 |
+
lines_by_y[round(w[1] / 5) * 5].append(w)
|
| 2274 |
+
rows_raw = [sorted(lines_by_y[y], key=lambda w: w[0])
|
| 2275 |
+
for y in sorted(lines_by_y)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2276 |
|
| 2277 |
+
if len(rows_raw) < 3:
|
| 2278 |
+
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2279 |
|
| 2280 |
+
# Column centers from first two header rows
|
| 2281 |
+
header_xs = sorted(w[0] for r in rows_raw[:2] for w in r)
|
| 2282 |
+
col_centers = []
|
| 2283 |
+
cluster = [header_xs[0]]
|
| 2284 |
+
for x in header_xs[1:]:
|
| 2285 |
+
if x - cluster[-1] > 30:
|
| 2286 |
+
col_centers.append(sum(cluster) / len(cluster))
|
| 2287 |
+
cluster = [x]
|
| 2288 |
+
else:
|
| 2289 |
+
cluster.append(x)
|
| 2290 |
+
col_centers.append(sum(cluster) / len(cluster))
|
| 2291 |
+
|
| 2292 |
+
def _assign(x):
|
| 2293 |
+
return min(range(len(col_centers)), key=lambda i: abs(col_centers[i] - x))
|
| 2294 |
+
|
| 2295 |
+
# Build column labels from first two rows
|
| 2296 |
+
col_labels = [""] * len(col_centers)
|
| 2297 |
+
for r in rows_raw[:2]:
|
| 2298 |
+
for w in r:
|
| 2299 |
+
ci = _assign(w[0])
|
| 2300 |
+
col_labels[ci] = (col_labels[ci] + " " + w[2]).strip()
|
| 2301 |
+
|
| 2302 |
+
# Parse data rows
|
| 2303 |
+
_MONEY = re.compile(r"^\$?-?[\d,]+\.\d{2}-?$")
|
| 2304 |
+
merged, cur_name, cur_vals = [], [], {}
|
| 2305 |
+
|
| 2306 |
+
for line in rows_raw[2:]:
|
| 2307 |
+
texts = [w[2] for w in line]
|
| 2308 |
+
has_money = any(_MONEY.match(t) for t in texts)
|
| 2309 |
+
is_totals = any(w[2].lower() == "totals" for w in line)
|
| 2310 |
+
|
| 2311 |
+
if has_money:
|
| 2312 |
+
for w in line:
|
| 2313 |
+
if not re.match(r"^\d{7,}$", w[2]) and w[2].lower() != "totals":
|
| 2314 |
+
cur_vals[_assign(w[0])] = w[2]
|
| 2315 |
+
acct_num = next((w[2] for w in line if re.match(r"^\d{7,}$", w[2])), "")
|
| 2316 |
+
name = " ".join(cur_name)
|
| 2317 |
+
if acct_num:
|
| 2318 |
+
name = (name + " " + acct_num).strip()
|
| 2319 |
+
if is_totals:
|
| 2320 |
+
name = "Totals"
|
| 2321 |
+
merged.append((name, dict(cur_vals)))
|
| 2322 |
+
cur_name, cur_vals = [], {}
|
| 2323 |
+
else:
|
| 2324 |
+
cur_name.extend(texts)
|
| 2325 |
|
| 2326 |
+
if not merged:
|
| 2327 |
+
return ""
|
| 2328 |
|
| 2329 |
+
n = len(col_labels)
|
| 2330 |
+
hdr = "<tr><th>Account</th>" + "".join(f"<th>{l}</th>" for l in col_labels) + "</tr>"
|
| 2331 |
+
rows_html = [
|
| 2332 |
+
"<tr><td>" + name + "</td>" +
|
| 2333 |
+
"".join(f"<td>{vals.get(i, '')}</td>" for i in range(n)) +
|
| 2334 |
+
"</tr>"
|
| 2335 |
+
for name, vals in merged
|
| 2336 |
]
|
| 2337 |
+
return (
|
| 2338 |
+
"Summary of your deposit accounts\n"
|
| 2339 |
+
"<table>\n" + hdr + "\n" +
|
| 2340 |
+
"\n".join(rows_html) + "\n</table>"
|
| 2341 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2342 |
|
| 2343 |
except Exception as e:
|
| 2344 |
+
log.warning("_extract_nfcu_summary_table failed (page %d): %s", page_num, e)
|
| 2345 |
return ""
|
| 2346 |
|
| 2347 |
def normalize_html_tables(text: str) -> str:
|
|
|
|
| 2509 |
try:
|
| 2510 |
path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
|
| 2511 |
is_pdf = path.lower().endswith(".pdf")
|
|
|
|
| 2512 |
parser = get_parser()
|
| 2513 |
|
| 2514 |
page_heights = []
|
|
|
|
| 2538 |
|
| 2539 |
parts = []
|
| 2540 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2541 |
# Header inclusion: PDF text -> band words -> OCR band
|
| 2542 |
hdr = ""
|
| 2543 |
if is_pdf:
|
|
|
|
| 2588 |
|
| 2589 |
parts.append(stabilized)
|
| 2590 |
|
| 2591 |
+
# Fix NF-1: prepend Navy Federal summary table from text layer.
|
| 2592 |
+
# Replaces the garbled OCR summary table with correct data.
|
| 2593 |
+
# Safe no-op for all non-NFCU PDFs (returns "" when guards fail).
|
| 2594 |
+
if is_pdf:
|
| 2595 |
+
try:
|
| 2596 |
+
nfcu_summary = _extract_nfcu_summary_table(path, page_num)
|
| 2597 |
+
if nfcu_summary:
|
| 2598 |
+
parts.insert(0, nfcu_summary)
|
| 2599 |
+
except Exception:
|
| 2600 |
+
pass
|
| 2601 |
+
|
| 2602 |
# Footer extraction: PDF text layer -> band words -> OCR crop
|
| 2603 |
# Deduplication guard prevents double-printing when body OCR already captured it.
|
| 2604 |
if ENABLE_FOOTER_OCR and page_num < len(page_images):
|