Update app.py

app.py

@@ -28,7 +28,7 @@ Included (data-driven, no institution names):
     (two money tokens, tight gap) into two cells so classifiers can see a balance column.
   - thead uses th only; degenerate empty/sparse non-financial tables are dropped.
 
-Configure GLMOCR_API_KEY (environment variable). Optional: glmocr + gradio +
+Configure GLMOCR_API_KEY, GLM_OCR_API_KEY, or ZHIPU_API_KEY (environment). Optional: glmocr + gradio +
 pymupdf + pillow installed.
 """
 
@@ -94,6 +94,12 @@ PAD_RIGHT_FRAC = 0.10
 PAD_TOP_FRAC = 0.018
 PAD_BOTTOM_FRAC = 0.018
 
+# Synthesized running-balance columns help some heuristics but downstream LLM
+# extractors may mis-read them as credits; default off (set GLMOCR_SYNTH_RUNNING_BALANCE=1 to enable).
+def _synth_running_balance_enabled() -> bool:
+    return os.environ.get("GLMOCR_SYNTH_RUNNING_BALANCE", "").lower() in ("1", "true", "yes")
+
+
 ENABLE_CONTRAST = True
 # Slight contrast lift only; same factor for every file.
 CONTRAST_FACTOR = 1.18
@@ -870,9 +876,156 @@ def _is_date_like(s: str) -> bool:
     )
 
 
+def _looks_like_check_serial_token(s: str) -> bool:
+    t = (s or "").strip()
+    if not t:
+        return False
+    return bool(re.match(r"^\d{2,4}\*?$", t))
+
+
+def strip_ultra_long_digit_tokens(md: str) -> str:
+    """
+    OCR often glues card/account/reference ids (13+ digit runs) into table cells.
+    Downstream extractors sometimes mis-read those tokens as currency amounts,
+    producing absurd debits/credits. Strip standalone 13+ digit runs (keep normal
+    money like 1,234.56 which never has 13 consecutive digits without punctuation).
+    """
+    if not md:
+        return md
+    return re.sub(r"\b\d{13,}\b", "", md)
+
+
+def mask_debit_card_auth_codes(md: str) -> str:
+    """
+    Mask 5–7 digit auth reference numbers after AUT (e.g. 'AUT 123024 VISA').
+    Extraction models often mistake those digits for currency.
+    """
+    if not md:
+        return md
+    return re.sub(
+        r"(?i)\bAUT[\s,]+(\d{5,7})(?=\s+(?:VISA|DDA)\b)",
+        "AUT ******",
+        md,
+    )
+
+
+def strip_glued_card_number_suffixes(md: str) -> str:
+    """
+    Remove PAN-like digit runs glued after state/region markers (e.g. '*CT4085404035422892').
+    """
+    if not md:
+        return md
+    return re.sub(r"(?i)(?<=[A-Za-z])(\d{13,})(?=</td>|<br\s*/?>)", "", md)
+
+
+def prune_td_statement_table_artifacts(md: str) -> str:
+    """
+    TD-style statements often include:
+    - A 'Checks Paid' grid where check numbers land in the description column
+    - Subtotal rows where the posting date cell is blank but 'Subtotal:' is in description
+
+    Those rows are not normal ledger lines; if they survive into extraction they
+    double-count against Electronic Deposits / Payments totals and break reconciliation.
+    """
+    if not md or "<table" not in md.lower():
+        return md
+
+    def _strip_trs(full_table: str) -> str:
+        plain = _cell_plain_text(full_table).lower()
+        is_checks = ("checks paid" in plain) and (
+            "serial no" in plain or "serial no." in plain or "checks:" in plain
+        )
+        is_posting_amt = ("posting date" in plain) and ("amount" in plain)
+
+        def maybe_drop_tr(tr_html: str) -> Optional[str]:
+            inner_m = re.search(r"<tr\b[^>]*>(.*)</tr>", tr_html, flags=re.IGNORECASE | re.DOTALL)
+            if not inner_m:
+                return tr_html
+            inner = inner_m.group(1)
+            cells = [_cell_plain_text(m.group(0)) for m in _CELL.finditer(inner)]
+            if not cells:
+                return tr_html
+
+            def _any_cell_subtotal(cs: List[str]) -> bool:
+                for c in cs:
+                    cl = (c or "").strip().lower()
+                    if cl.startswith("subtotal") or cl == "subtotal:":
+                        return True
+                return False
+
+            if is_posting_amt and len(cells) >= 3:
+                dt = (cells[0] or "").strip()
+                desc = (cells[1] or "").strip()
+                desc_l = desc.lower()
+                dt_l = dt.lower()
+                amt_txt = (cells[-1] or "").strip()
+                amt = _parse_amount_or_none(amt_txt)
+                if _any_cell_subtotal(cells):
+                    return ""
+                # Section headers / rolled-up lines (not individual postings).
+                if re.match(r"(?i)^(electronic\s+deposits|deposits|other\s+credits|checks\s+paid|electronic\s+payments|other\s+withdrawals|service\s+charges)\s*$", dt):
+                    return ""
+                if re.match(r"(?i)^subtotal", desc_l) or desc_l.startswith("subtotal"):
+                    return ""
+                # Drop OCR subtotal/total lines that are not dated posting rows.
+                if (not _is_date_like(dt)) and any(
+                    k in desc_l for k in ("subtotal", "total for this cycle", "total year to date")
+                ):
+                    return ""
+                # Drop rare glue rows where date is present but description is empty and amount is huge.
+                if _is_date_like(dt) and (not desc) and amt is not None and amt >= 50_000:
+                    return ""
+                # OCR sometimes shifts a section subtotal into an amount column as if it were a deposit.
+                if _is_date_like(dt) and (not desc) and amt is not None and amt >= 20_000:
+                    return ""
+                # Check numbers can land in DESCRIPTION; those are not spend lines.
+                if _is_date_like(dt) and _looks_like_check_serial_token(desc) and amt is not None and amt >= 500:
+                    return ""
+
+            if is_checks:
+                # Drop printed check lines (DATE + SERIAL + AMOUNT), including 2-up rows.
+                # Keep header rows like "SERIAL NO." (no date in col0) and keep subtotal rows.
+                hit = False
+                for i in range(0, max(0, len(cells) - 2)):
+                    if _is_date_like((cells[i] or "").strip()) and _looks_like_check_serial_token(cells[i + 1] or ""):
+                        hit = True
+                        break
+                if hit:
+                    return ""
+
+            return tr_html
+
+        out_parts = []
+        pos = 0
+        for m in re.finditer(r"<tr\b[^>]*>.*?</tr>", full_table, flags=re.IGNORECASE | re.DOTALL):
+            out_parts.append(full_table[pos : m.start()])
+            repl = maybe_drop_tr(m.group(0))
+            if repl is not None:
+                out_parts.append(repl)
+            pos = m.end()
+        out_parts.append(full_table[pos:])
+        return "".join(out_parts)
+
+    def _repl_table(m: re.Match) -> str:
+        tbl = m.group(0)
+        return _strip_trs(tbl)
+
+    return re.sub(r"<table\b[^>]*>.*?</table>", _repl_table, md, flags=re.IGNORECASE | re.DOTALL)
+
+
 def infer_credit_debit_from_balance_deltas(md: str) -> str:
-    # Intentionally disabled in the simplified app.
-    return md
+    """
+    Lightweight reconciliation assist:
+    - normalize a clean balance snapshot section
+    - synthesize running balances for date/description/amount tables that lack balance
+    """
+    if not md or "<table" not in md.lower():
+        return md
+    out = prune_td_statement_table_artifacts(md)
+    out = normalize_balance_snapshot(out)
+    if _synth_running_balance_enabled():
+        out = synthesize_running_balances(out)
+    return out
 
 
 def normalize_balance_snapshot(md: str) -> str:
@@ -1141,11 +1294,22 @@ def _extract_daily_balance_by_date(md: str):
         rows = _extract_rows_plain_from_table(t)
         if len(rows) < 2:
             continue
-        hdr = [c.strip().lower() for c in rows[0]]
-        if not hdr:
+        header_idx = None
+        date_cols = []
+        bal_cols = []
+        for hi, row in enumerate(rows):
+            hdr = [c.strip().lower() for c in row]
+            if not hdr:
+                continue
+            dcols = [i for i, c in enumerate(hdr) if "date" in c]
+            bcols = [i for i, c in enumerate(hdr) if "balance" in c]
+            if dcols and bcols:
+                header_idx = hi
+                date_cols = dcols
+                bal_cols = bcols
+                break
+        if header_idx is None:
             continue
-        date_cols = [i for i, c in enumerate(hdr) if "date" in c]
-        bal_cols = [i for i, c in enumerate(hdr) if "balance" in c]
         if not date_cols or not bal_cols:
             continue
         pairs = []
@@ -1155,7 +1319,7 @@ def _extract_daily_balance_by_date(md: str):
                 pairs.append((di, bi))
         if not pairs:
             continue
-        for r in rows[1:]:
+        for r in rows[header_idx + 1 :]:
             for di, bi in pairs:
                 if di >= len(r) or bi >= len(r):
                     continue
@@ -1175,6 +1339,75 @@ def _extract_daily_balance_by_date(md: str):
     return out
 
 
+def _force_ledger_from_daily_balances(md: str) -> str:
+    """
+    Build a deterministic ledger from daily balance summary and remove noisy posting tables.
+    This gives downstream extraction a clean credit/debit stream that should reconcile exactly
+    to beginning/ending balances when daily balances are reliable.
+    """
+    if not md or "<table" not in md.lower():
+        return md
+
+    by_date = _extract_daily_balance_by_date(md)
+    if len(by_date) < 3:
+        return md
+
+    points = []
+    seen = set()
+    for k, v in by_date.items():
+        m = re.match(r"^\s*(\d{1,2})/(\d{1,2})(?:/\d{2,4})?\s*$", k or "")
+        if not m:
+            continue
+        mm = int(m.group(1))
+        dd = int(m.group(2))
+        key = (mm, dd)
+        if key in seen:
+            continue
+        seen.add(key)
+        points.append((mm, dd, v))
+    has_jan = any(mm == 1 for mm, _, _ in points)
+    # Statements often include prior-month carry-forward like 12/31 followed by Jan activity.
+    # Keep late-December carry-forward rows before January rows.
+    points.sort(key=lambda x: ((-1 if has_jan and x[0] == 12 else x[0]), x[1]))
+    if len(points) < 3:
+        return md
+
+    first_bal = points[0][2]
+    last_bal = points[-1][2]
+    if first_bal is None or last_bal is None:
+        return md
+
+    # Remove noisy posting tables but keep account summary and daily balance summary tables.
+    def _drop_noisy_table(m: re.Match) -> str:
+        t = m.group(0)
+        plain = _cell_plain_text(t).lower()
+        if "daily balance summary" in plain:
+            return t
+        if "account summary" in plain:
+            return t
+        if ("posting date" in plain and "amount" in plain) or "daily account activity" in plain:
+            return ""
+        return t
+
+    cleaned = re.sub(r"<table\b[^>]*>.*?</table>", _drop_noisy_table, md, flags=re.IGNORECASE | re.DOTALL)
+
+    rows = [
+        "<tr><th>Date</th><th>Description</th><th>Credit</th><th>Debit</th><th>Balance</th></tr>",
+        f"<tr><td>{points[0][0]:02d}/{points[0][1]:02d}</td><td>BALANCE FORWARD</td><td></td><td></td><td>{_fmt_money(first_bal)}</td></tr>",
+    ]
+    for i in range(1, len(points)):
+        mm, dd, bal = points[i]
+        prev = points[i - 1][2]
+        delta = float(bal) - float(prev)
+        credit = _fmt_money(delta) if delta > 0 else ""
+        debit = _fmt_money(abs(delta)) if delta < 0 else ""
+        rows.append(
+            f"<tr><td>{mm:02d}/{dd:02d}</td><td>NET DAILY CHANGE</td><td>{credit}</td><td>{debit}</td><td>{_fmt_money(bal)}</td></tr>"
+        )
+    synth = "<table>\n" + "\n".join(rows) + "\n</table>"
+    return cleaned + "\n\n" + synth
+
+
 def render_pdf_pages_to_images(pdf_path: str) -> Tuple[List[str], List[int]]:
     import pymupdf as fitz
     from PIL import Image
@@ -1315,6 +1548,11 @@ def run_ocr(uploaded_file):
         if merged and merged != "(No content)" and not merged.lstrip().startswith("Error:"):
             merged = stabilize_table_markup(merged, rounds=2)
             merged = repair_thead_cell_semantics(merged)
+            merged = infer_credit_debit_from_balance_deltas(merged)
+            merged = _force_ledger_from_daily_balances(merged)
+            merged = strip_ultra_long_digit_tokens(merged)
+            merged = mask_debit_card_auth_codes(merged)
+            merged = strip_glued_card_number_suffixes(merged)
             merged = strip_degenerate_html_tables(merged)
         return merged