Update app.py

app.py

@@ -2191,158 +2191,301 @@ def convert_plaintext_bank_sections(text: str) -> str:
     return "\n".join(out)
 
 # --------------------------
-# Navy Federal Summary Table extraction (Fix NF-1)
+# Navy Federal full-page text-layer parser (Fix NF-1)
 # --------------------------
 
-def _extract_nfcu_summary_table(pdf_path: str, page_num: int) -> str:
+def _parse_nfcu_page(pdf_path: str, page_num: int) -> str:
     """
-    Extract Navy Federal 'Summary of your deposit accounts' table from the PDF
-    text layer using spatial word positions.
+    Full text-layer parser for Navy Federal Credit Union statements.
 
-    GLM-OCR fails on this table because the five data columns are spread very
-    wide across the page and the account-name column has no visible border.
-    The PDF text layer contains the data perfectly — we read it with pdfplumber.
+    GLM-OCR fails on NFCU pages because:
+      - The summary table has 5 wide columns with no visible borders
+      - Transaction tables have Amount($) and Balance($) spatially far right
+      - Multi-line descriptions (wrapped lines) confuse OCR table detection
+
+    This function reads the PDF text layer directly via pdfplumber and produces
+    correct, complete HTML for every section: summary table, transaction tables,
+    items paid, savings, disclosures.
 
     Detection guards (all must pass — safe no-op for all other banks):
-      1. A word 'Summary' must appear in the top 55% of the page height.
-      2. Words 'deposit' or 'accounts' must appear within 15pt of that word.
+      1. Page text must contain 'Access No.' AND 'Statement of Account'
+         (unique to Navy Federal statement format)
+      2. pdfplumber must return extractable text (not a scanned page)
 
-    Returns HTML table string, or "" if not found or on any error.
+    Returns complete HTML string for the page, or "" if not NFCU / any error.
     """
     try:
         import pdfplumber
-        from collections import defaultdict as _defaultdict
 
         with pdfplumber.open(pdf_path) as pdf:
             if page_num >= len(pdf.pages):
                 return ""
             page = pdf.pages[page_num]
-            words = page.extract_words()
-            page_h = float(page.height)
-
-            # Guard 1: find 'Summary' word in top 55% of page
-            summary_word = next(
-                (w for w in words
-                 if w['text'].lower() == 'summary' and w['top'] < page_h * 0.55),
-                None
-            )
-            if not summary_word:
-                return ""
+            text = page.extract_text() or ""
 
-            # Guard 2: nearby words must confirm this is a deposit summary
-            nearby = [w['text'].lower() for w in words
-                      if abs(w['top'] - summary_word['top']) < 15]
-            if 'deposit' not in nearby and 'accounts' not in nearby:
-                return ""
+        if not text:
+            return ""
 
-            # Band: start ~15pt below "Summary of..." heading, span ~115pt
-            y_start = summary_word['top'] + 15
-            y_end   = summary_word['top'] + 115
+        # Guard: must match NFCU format on this page OR on page 0
+        # (pages 2+ still have "Access No." and "Statement of Account")
+        is_nfcu = (
+            ('Access No.' in text and 'Statement of Account' in text) or
+            ('navyfederal' in text.lower()) or
+            ('Navy Federal' in text)
+        )
+        if not is_nfcu:
+            return ""
 
-            band_words = [w for w in words if y_start <= w['top'] <= y_end]
-            if not band_words:
-                return ""
+        lines = [l.strip() for l in text.splitlines()]
 
-            # Group words into lines by y-bucket (5pt tolerance)
-            lines_by_y = _defaultdict(list)
-            for w in band_words:
-                bucket = round(w['top'] / 5) * 5
-                lines_by_y[bucket].append(w)
+        # ── Regex patterns ─────────────────────────────────────────────────
+        # Full transaction: "02-03 Description 7.68- 3,974.66"
+        TXN_FULL = re.compile(
+            r'^(\d{2}-\d{2})\s+(.+?)\s+([\d,]+\.\d{2}-?)\s+([\d,]+\.\d{2}-?)$'
+        )
+        # Transaction with continuation: next line has "Location amt bal"
+        TXN_NOAMT = re.compile(r'^(\d{2}-\d{2})\s+(.+)$')
+        # Continuation line ending with amount + balance
+        CONT_MONEY = re.compile(r'^(.+?)\s+([\d,]+\.\d{2}-?)\s+([\d,]+\.\d{2}-?)$')
+        # Items Paid two-column row
+        ITEMS_2COL = re.compile(
+            r'^(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})'
+            r'\s+(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})$'
+        )
+        ITEMS_1COL = re.compile(
+            r'^(\d{2}-\d{2})\s+(ACH|POS|CHECK|ATM|WIRE)\s+([\d,]+\.\d{2})$'
+        )
+        BOILERPLATE = re.compile(
+            r'^(Page \d+ of \d+|Access No\.\s*\d+|Statement Period'
+            r'|\d{2}/\d{2}/\d{2}\s*-\s*\d{2}/\d{2}/\d{2}'
+            r'|Statement of Account|For VIRTUAL EDUCATION STATION'
+            r'|PO Box \d+.*|navyfederal\.org.*'
+            r'|Say\s*"?Yes"?\s*to Paperless'
+            r'|Insured\s*by\s*NCUA.*|InsuredbyNCUA.*'
+            r'|IfyouhavenT?.*|Togetstarted.*|It\'saneasy.*'
+            r'|digital statements via Mobile'
+            r'|Navy Federal Online Banking\.)$'
+        )
+        SKIP_LINE = re.compile(
+            r'^(REMITTANCE RECEIVED|VIRTUAL EDUCATION STATION|18511991'
+            r'|DEPOSIT\s*VOUCHER|FOR MAIL USE ONLY|DEPOSITS MAY NOT'
+            r'|ACCOUNT\s*NUMBER|MARK\s*"?X"?\s*TO CHANGE|ADDRESS/ORDER'
+            r'|ITEMS ON REVERSE|NFCU|PO\s*BOX\s*3100|MERRIFIELD'
+            r'|405715|CHANGE OF ADDRESS|PLEASE PRINT'
+            r'|RANK/RATE|ADDRESS\(NO|CITY|STATE|ZIP|SIGNATURE'
+            r'|EFFECTIVE DATE|HOME TELEPHONE|DAYTIME TELEPHONE'
+            r'|- -|\( \)|Say Yes to Paperless).*$',
+            re.IGNORECASE
+        )
 
-            rows_raw = [sorted(lines_by_y[y], key=lambda w: w['x0'])
-                        for y in sorted(lines_by_y.keys())]
+        # ── Helper: parse transaction block ────────────────────────────────
+        def parse_txn_block(i, stop_words):
+            rows = []
+            while i < len(lines):
+                l = lines[i]
+                if not l:
+                    i += 1; continue
+                if BOILERPLATE.match(l) or SKIP_LINE.match(l):
+                    i += 1; continue
+                if any(l == sw or l.startswith(sw) for sw in stop_words):
+                    break
 
-            if len(rows_raw) < 3:
-                return ""
+                # Full single-line transaction
+                m = TXN_FULL.match(l)
+                if m:
+                    date, desc, amt, bal = m.group(1), m.group(2), m.group(3), m.group(4)
+                    rows.append(f'<tr><td>{date}</td><td>{desc}</td><td>{amt}</td><td>{bal}</td></tr>')
+                    i += 1
+                    continue
 
-            # Column centers from first two header rows
-            # (two-line header: "Previous Deposits/ Withdrawals/ Ending YTD" /
-            #                    "Balance  Credits   Debits       Balance Dividends")
-            header_words_flat = [w for r in rows_raw[:2] for w in r]
-            header_xs = sorted((w['x0'] + w['x1']) / 2 for w in header_words_flat)
-
-            col_centers = []
-            cluster = [header_xs[0]]
-            for x in header_xs[1:]:
-                if x - cluster[-1] > 30:
-                    col_centers.append(sum(cluster) / len(cluster))
-                    cluster = [x]
-                else:
-                    cluster.append(x)
-            col_centers.append(sum(cluster) / len(cluster))
-
-            def _assign_col(x_center):
-                return min(range(len(col_centers)),
-                           key=lambda i: abs(col_centers[i] - x_center))
-
-            # Build column labels by merging two header rows
-            col_labels = [''] * len(col_centers)
-            for r in rows_raw[:2]:
-                for w in r:
-                    xc = (w['x0'] + w['x1']) / 2
-                    ci = _assign_col(xc)
-                    sep = ' ' if col_labels[ci] else ''
-                    col_labels[ci] += sep + w['text']
-
-            # Data rows: each account has a name line + a values line
-            _MONEY_PAT = re.compile(r'^\$[\d,]+\.?\d*-?$|^[\d,]+\.?\d*-?$')
-
-            merged = []
-            current_name = []
-            current_vals = {}
-
-            for line in rows_raw[2:]:
-                texts = [w['text'] for w in line]
-                has_money = any(_MONEY_PAT.match(t) for t in texts)
-                is_totals = any(w['text'].lower() == 'totals' for w in line)
-
-                if has_money:
-                    # Values line — may also contain account number and/or 'Totals'
-                    val_words = [w for w in line
-                                 if not re.match(r'^\d{7,}$', w['text'])
-                                 and w['text'].lower() != 'totals']
-                    acct_words = [w for w in line if re.match(r'^\d{7,}$', w['text'])]
-
-                    for w in val_words:
-                        xc = (w['x0'] + w['x1']) / 2
-                        ci = _assign_col(xc)
-                        current_vals[ci] = w['text']
-
-                    name_str = ' '.join(current_name)
-                    if acct_words:
-                        name_str = (name_str + ' ' + acct_words[0]['text']).strip()
-                    if is_totals:
-                        name_str = 'Totals'
-
-                    merged.append((name_str, dict(current_vals)))
-                    current_name = []
-                    current_vals = {}
-                else:
-                    current_name.extend(texts)
+                # Multi-line: first line has date + partial desc, next has rest+amt+bal
+                mn = TXN_NOAMT.match(l)
+                if mn:
+                    date, desc1 = mn.group(1), mn.group(2)
+                    i += 1
+                    if i < len(lines):
+                        nxt = lines[i]
+                        mc = CONT_MONEY.match(nxt)
+                        if mc:
+                            desc2, amt, bal = mc.group(1), mc.group(2), mc.group(3)
+                            full_desc = (desc1 + ' ' + desc2).strip()
+                            rows.append(f'<tr><td>{date}</td><td>{full_desc}</td><td>{amt}</td><td>{bal}</td></tr>')
+                            i += 1
+                            continue
+                        # Next line is not a continuation — treat as balance-only
+                        rows.append(f'<tr><td>{date}</td><td>{desc1}</td><td></td><td></td></tr>')
+                    continue
 
-            if not merged:
-                return ""
+                i += 1
+            return rows, i
+
+        # ── Main parse loop ────────────────────────────────────────────────
+        out = []
+        i = 0
+        TXN_STOP = [
+            'Average Daily Balance', 'Items Paid', 'Savings', 'Checking',
+            'Mbr Business Savings -', '2024 Year to Date', 'Your share balance',
+            'Disclosure', 'What to Do', 'Errors Related', 'Errors Within', 'Payments'
+        ]
+
+        while i < len(lines):
+            l = lines[i]
+
+            if not l or BOILERPLATE.match(l) or SKIP_LINE.match(l):
+                i += 1; continue
+
+            # ── Summary of your deposit accounts ──────────────────────────
+            if l == 'Summary of your deposit accounts':
+                i += 1
+                # Skip the two header lines ("Previous Deposits/..." and "Balance Credits...")
+                while i < len(lines) and not re.match(r'^(Business|Mbr|Totals)', lines[i]):
+                    i += 1
+
+                out.append('<b>Summary of your deposit accounts</b>')
+                out.append('<table>')
+                out.append('<tr><th>Account</th><th>Previous Balance</th>'
+                           '<th>Deposits/Credits</th><th>Withdrawals/Debits</th>'
+                           '<th>Ending Balance</th><th>YTD Dividends</th></tr>')
 
-            n_cols = len(col_labels)
-            header_html = '<tr><th>Account</th>' + ''.join(
-                f'<th>{lbl}</th>' for lbl in col_labels) + '</tr>'
-
-            rows_html = []
-            for name, vals in merged:
-                row = f'<tr><td>{name}</td>'
-                for ci in range(n_cols):
-                    row += f'<td>{vals.get(ci, "")}</td>'
-                row += '</tr>'
-                rows_html.append(row)
-
-            return (
-                'Summary of your deposit accounts\n'
-                '<table>\n' + header_html + '\n' +
-                '\n'.join(rows_html) + '\n</table>'
+                while i < len(lines):
+                    l2 = lines[i]
+                    if not l2 or re.match(r'^(Checking$|#|REMITTANCE)', l2):
+                        break
+
+                    if l2.startswith('Totals '):
+                        parts = l2.split()
+                        vals = parts[1:]
+                        while len(vals) < 5: vals.append('')
+                        cells = ''.join(f'<td>{v}</td>' for v in vals[:5])
+                        out.append(f'<tr><td><b>Totals</b></td>{cells}</tr>')
+                        i += 1
+                        break
+
+                    if re.match(r'^(Business Checking|Mbr Business Savings)', l2):
+                        acct_name = l2
+                        i += 1
+                        if i < len(lines):
+                            val_line = lines[i]
+                            parts = val_line.split()
+                            if parts and re.match(r'^\d{7,}$', parts[0]):
+                                acct_num = parts[0]
+                                vals = parts[1:]
+                            else:
+                                acct_num = ''
+                                vals = parts
+                            while len(vals) < 5: vals.append('')
+                            cells = ''.join(f'<td>{v}</td>' for v in vals[:5])
+                            name_cell = f'{acct_name} ({acct_num})' if acct_num else acct_name
+                            out.append(f'<tr><td>{name_cell}</td>{cells}</tr>')
+                            i += 1
+                        continue
+                    i += 1
+
+                out.append('</table>')
+                continue
+
+            # ── Section headers ────────────────────────────────────────────
+            if l in ('Checking', 'Savings'):
+                out.append(f'<h3>{l}</h3>')
+                i += 1
+                continue
+
+            # ── Account transaction table ────────────────────────��─────────
+            m_acct = re.match(
+                r'^(Business Checking|Mbr Business Savings) - (\d+)(.*)?$', l
             )
+            if m_acct:
+                acct_line = l
+                i += 1
+                if i < len(lines) and 'Continued' in lines[i]:
+                    acct_line += ' ' + lines[i]
+                    i += 1
+                if i < len(lines) and lines[i].startswith('Date '):
+                    i += 1  # skip "Date Transaction Detail Amount($) Balance($)"
+
+                out.append(f'<b>{acct_line}</b>')
+                out.append('<table>')
+                out.append('<tr><th>Date</th><th>Transaction Detail</th>'
+                           '<th>Amount($)</th><th>Balance($)</th></tr>')
+
+                rows, i = parse_txn_block(i, TXN_STOP)
+                out.extend(rows)
+                out.append('</table>')
+                continue
+
+            # ── Items Paid ─────────────────────────────────────────────────
+            if l == 'Items Paid' or l.startswith('Items Paid (Continued'):
+                out.append(f'<b>{l}</b>')
+                out.append('<table>')
+                out.append('<tr><th>Date</th><th>Item</th><th>Amount($)</th>'
+                           '<th>Date</th><th>Item</th><th>Amount($)</th></tr>')
+                i += 1
+                if i < len(lines) and lines[i].startswith('Date '):
+                    i += 1
+
+                ITEMS_STOP = ['Savings', '2024 Year to Date', 'Checking',
+                              'Disclosure', 'What to Do']
+                while i < len(lines):
+                    l2 = lines[i]
+                    if not l2: i += 1; continue
+                    if any(l2 == sw or l2.startswith(sw) for sw in ITEMS_STOP):
+                        break
+                    if BOILERPLATE.match(l2) or SKIP_LINE.match(l2):
+                        i += 1; continue
+
+                    m2c = ITEMS_2COL.match(l2)
+                    if m2c:
+                        d1,t1,a1,d2,t2,a2 = m2c.groups()
+                        out.append(f'<tr><td>{d1}</td><td>{t1}</td><td>{a1}</td>'
+                                   f'<td>{d2}</td><td>{t2}</td><td>{a2}</td></tr>')
+                        i += 1; continue
+
+                    m1c = ITEMS_1COL.match(l2)
+                    if m1c:
+                        d1,t1,a1 = m1c.groups()
+                        out.append(f'<tr><td>{d1}</td><td>{t1}</td><td>{a1}</td>'
+                                   f'<td></td><td></td><td></td></tr>')
+                        i += 1; continue
+                    i += 1
+
+                out.append('</table>')
+                continue
+
+            # ── Average Daily Balance ──────────────────────────────────────
+            if l.startswith('Average Daily Balance'):
+                out.append(f'<p>{l}</p>')
+                i += 1; continue
+
+            # ── 2024 Year to Date ──────────────────────────────────────────
+            if l.startswith('2024 Year to Date'):
+                out.append(f'<p><b>{l}</b></p>')
+                i += 1
+                while i < len(lines) and lines[i] and not BOILERPLATE.match(lines[i]):
+                    out.append(f'<p>{lines[i]}</p>')
+                    i += 1
+                continue
+
+            # ── Your share balance is overdrawn ───────────────────────────
+            if l.startswith('Your share balance'):
+                out.append(f'<p><i>{l}</i></p>')
+                i += 1; continue
+
+            # ── Disclosure page — output as plain text ────────────────────
+            if any(l.startswith(x) for x in
+                   ('Disclosure', 'WhattoDoif', 'What to Do', 'Errors', 'Payments',
+                    'ErrorsRelated', 'ErrorsWithin')):
+                out.append(f'<p>{l}</p>')
+                i += 1; continue
+
+            # Default: keep non-empty lines
+            if l:
+                out.append(f'<p>{l}</p>')
+            i += 1
+
+        return '\n'.join(out)
 
     except Exception as e:
-        log.warning("_extract_nfcu_summary_table failed (page %d): %s", page_num, e)
+        log.warning("_parse_nfcu_page failed (page %d): %s", page_num, e)
         return ""
 
 def normalize_html_tables(text: str) -> str:
@@ -2589,17 +2732,21 @@ def run_ocr(uploaded_file):
 
                 parts.append(stabilized)
 
-            # Fix NF-1: prepend Navy Federal "Summary of your deposit accounts" table.
-            # Uses pdfplumber spatial word positions to extract the wide-column summary
-            # table that GLM-OCR fails to parse correctly.
-            # Safe no-op for all non-NFCU PDFs (returns "" when guards don't match).
+            # Fix NF-1: full text-layer parse for Navy Federal statements.
+            # GLM-OCR fails on NFCU pages (summary table, transaction columns,
+            # multi-line descriptions). When NFCU is detected, replace the OCR
+            # body with a clean text-layer extraction and clear the parts list
+            # to avoid mixing garbled OCR output with correct text-layer data.
+            # Safe no-op for all non-NFCU PDFs (_parse_nfcu_page returns "").
             if is_pdf:
                 try:
-                    nfcu_summary = _extract_nfcu_summary_table(path, page_num)
-                    if nfcu_summary:
-                        parts.insert(0, nfcu_summary)
+                    nfcu_html = _parse_nfcu_page(path, page_num)
+                    if nfcu_html:
+                        # Replace ALL parts with the text-layer result —
+                        # discard garbled OCR header/body/footer for this page
+                        parts = [nfcu_html]
                 except Exception:
-                    pass
+                    pass  # safe no-op — falls back to OCR output on any error
 
             # Footer extraction: PDF text layer -> band words -> OCR crop
             # Deduplication guard prevents double-printing when body OCR already captured it.