Update app.py

app.py

@@ -113,6 +113,12 @@ Primary goals for reconcile rate:
             Additions/Subtractions tables to the PDF: credits as Number|Date|Description|Additions
             (blank Number when unused); debits as Date|Description|Subtractions (no Number); merges
             OCR column splits. Adjacent duplicate HTML tables (same header + row multiset) are removed.
+- Fix TD-1: TD Bank continuation register: (a) four OCR columns with DATE+DESCRIPTION fused +
+            DEBIT/CREDIT/BALANCE — split col0 into Date|Description; (b) three OCR columns
+            Date|Transaction Description|Amount where Amount is the running balance — expand to
+            five columns using trailing amount in the description cell and TD-specific debit/credit
+            routing. Fires on \"continued\" in-table text and/or TD body cues (TD ZELLE, LENDINGCLUB,
+            etc.). Safe no-op elsewhere.
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -2867,6 +2873,216 @@ def _normalize_fused_date_posted_amount_table(grid):
         return grid
     return out
 
+
+_TD_CONTINUED_BLOB_RE = re.compile(
+    r"transactions\s+by\s+date.*continued|continued.*transactions\s+by\s+date",
+    re.IGNORECASE | re.DOTALL,
+)
+
+# GLM often omits the continued banner inside a standalone <table>; match TD register body text.
+_TD_REGISTER_BODY_SIGNAL_RE = re.compile(
+    r"TD\s+ZELLE|\bZELLE\b.*\b(?:RECEIVED|SENT)\b|LENDINGCLUB|MM\s*&\s*(?:amp;)?\s*CB|"
+    r"MAINTENANCE\s+FEE|OD\s+GRACE\s+FEE\s+REFUND|OVERDRAFT\s+PD",
+    re.IGNORECASE,
+)
+
+
+def _is_td_four_col_ddcb_header(cells: List[str]) -> bool:
+    """TD continuation: one fused DATE DESCRIPTION cell + DEBIT + CREDIT + BALANCE."""
+    if len(cells) < 4:
+        return False
+    a = re.sub(r"\s+", " ", str(cells[0] or "").strip().lower())
+    b = re.sub(r"\s+", " ", str(cells[1] or "").strip().lower())
+    c = re.sub(r"\s+", " ", str(cells[2] or "").strip().lower())
+    d = re.sub(r"\s+", " ", str(cells[3] or "").strip().lower())
+    fused = "date" in a and "description" in a
+    deb = b == "debit" or b.startswith("debit")
+    cre = "credit" in c and "debit" not in c
+    bal = "balance" in d
+    return fused and deb and cre and bal
+
+
+def _split_td_continuation_date_description_grid(grid):
+    """
+    Fix TD-1: TD Convenience Checking continuation pages use four physical columns; split col0
+    into Date + Description so json_data_extractor's five-way column map aligns with row cells.
+    """
+    if not grid or len(grid) < 3:
+        return grid
+
+    blob = " ".join(
+        " ".join(str(c or "") for c in row) for row in grid[: min(30, len(grid))]
+    )
+    if not _TD_CONTINUED_BLOB_RE.search(blob):
+        return grid
+
+    hdr_idx = None
+    for ri in range(min(20, len(grid))):
+        raw = [str(c or "").strip() for c in grid[ri]]
+        while len(raw) < 4:
+            raw.append("")
+        if len(raw) >= 4 and _is_td_four_col_ddcb_header(raw[:4]):
+            hdr_idx = ri
+            break
+    if hdr_idx is None:
+        return grid
+
+    date_led_re = re.compile(r"^(\d{1,2}/\d{1,2}(?:/\d{2,4})?)\s+(.*)$", re.DOTALL)
+    new_grid: List[List[str]] = []
+    for ri, row in enumerate(grid):
+        cells = [str(c or "").strip() for c in row]
+        if ri < hdr_idx:
+            new_grid.append(cells)
+            continue
+        if ri == hdr_idx:
+            c = cells[:4] + [""] * max(0, 4 - len(cells))
+            new_grid.append(
+                [
+                    "DATE",
+                    "DESCRIPTION",
+                    c[1] or "DEBIT",
+                    c[2] or "CREDIT",
+                    c[3] or "BALANCE",
+                ]
+            )
+            continue
+        while len(cells) < 4:
+            cells.append("")
+        if len(cells) == 4:
+            m = date_led_re.match(cells[0].strip())
+            if m:
+                new_grid.append(
+                    [
+                        m.group(1).strip(),
+                        (m.group(2) or "").strip(),
+                        cells[1],
+                        cells[2],
+                        cells[3],
+                    ]
+                )
+            else:
+                new_grid.append([cells[0], "", cells[1], cells[2], cells[3]])
+        elif len(cells) >= 5:
+            new_grid.append(cells)
+        else:
+            while len(cells) < 5:
+                cells.append("")
+            new_grid.append(cells[:5])
+
+    return new_grid
+
+
+def _is_td_three_col_register_header(cells: List[str]) -> bool:
+    """OCR variant: Date | Transaction Description | Amount (Amount column is running balance)."""
+    if len(cells) < 3:
+        return False
+    a = re.sub(r"\s+", " ", str(cells[0] or "").strip().lower())
+    b = re.sub(r"\s+", " ", str(cells[1] or "").strip().lower())
+    c = re.sub(r"\s+", " ", str(cells[2] or "").strip().lower())
+    if a != "date":
+        return False
+    if "transaction" not in b or "description" not in b:
+        return False
+    if c not in ("amount", "balance") and "amount" not in c and "balance" not in c:
+        return False
+    return True
+
+
+def _td_trailing_amount_from_description(desc: str) -> Tuple[str, str]:
+    """Return (description_without_trailing_amount, amount_str or '')."""
+    if not desc or not str(desc).strip():
+        return "", ""
+    s = str(desc).strip()
+    m = re.search(r"(?:,\s*|\s)(-?\d{1,3}(?:,\d{3})*\.\d{2})\s*$", s)
+    if not m:
+        return s, ""
+    amt = m.group(1).strip()
+    left = s[: m.start()].strip().rstrip(",")
+    return left, amt
+
+
+def _td_route_debit_credit(desc_raw: str, amt_str: str) -> Tuple[str, str]:
+    """Return (debit_cell, credit_cell) with positive amount string in one side."""
+    if not amt_str:
+        return "", ""
+    u = str(desc_raw or "").upper()
+    if (
+        "ACH DEBIT" in u
+        or "OVERDRAFT" in u
+        or "ZELLE SENT" in u
+        or "MAINTENANCE FEE" in u
+        or "ATM DB" in u
+        or "ATM DEBIT" in u
+        or "ELECTRONIC PMT" in u
+        or "PMT-WEB" in u
+    ):
+        return amt_str, ""
+    if (
+        "ZELLE RECEIVED" in u
+        or "OD GRACE FEE REFUND" in u
+        or u.startswith("CREDIT,")
+        or u.startswith("DEPOSIT")
+    ):
+        return "", amt_str
+    if "DEBIT" in u and "CREDIT," not in u[:12]:
+        return amt_str, ""
+    return "", amt_str
+
+
+def _expand_td_continuation_three_col_register_grid(grid):
+    """
+    Fix TD-1b: GLM emits Date|Transaction Description|Amount where col3 is balance; expand to
+    DATE|DESCRIPTION|DEBIT|CREDIT|BALANCE for json_data_extractor.
+    """
+    if not grid or len(grid) < 2:
+        return grid
+
+    blob = " ".join(
+        " ".join(str(c or "") for c in row) for row in grid[: min(40, len(grid))]
+    )
+    if not (
+        _TD_CONTINUED_BLOB_RE.search(blob)
+        or _TD_REGISTER_BODY_SIGNAL_RE.search(blob)
+    ):
+        return grid
+
+    hdr_idx = None
+    for ri in range(min(15, len(grid))):
+        raw = [str(c or "").strip() for c in grid[ri]]
+        while len(raw) < 3:
+            raw.append("")
+        if _is_td_three_col_register_header(raw[:3]):
+            hdr_idx = ri
+            break
+    if hdr_idx is None:
+        return grid
+
+    date_re = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
+    new_grid: List[List[str]] = []
+    for ri, row in enumerate(grid):
+        cells = [str(c or "").strip() for c in row]
+        if ri < hdr_idx:
+            new_grid.append(cells)
+            continue
+        if ri == hdr_idx:
+            new_grid.append(["DATE", "DESCRIPTION", "DEBIT", "CREDIT", "BALANCE"])
+            continue
+        while len(cells) < 3:
+            cells.append("")
+        if len(cells) != 3:
+            new_grid.append(cells)
+            continue
+        d0, d1, bal = cells[0], cells[1], cells[2]
+        if not date_re.match(d0):
+            new_grid.append([d0, d1, "", "", bal])
+            continue
+        desc_only, amt_s = _td_trailing_amount_from_description(d1)
+        db, cr = _td_route_debit_credit(d1, amt_s)
+        new_grid.append([d0, desc_only, db, cr, bal])
+
+    return new_grid
+
+
 def normalize_html_tables(text: str) -> str:
     if not text or "<table" not in text.lower():
         return text
@@ -2908,6 +3124,9 @@ def normalize_html_tables(text: str) -> str:
 
             grid = _split_fused_date_transaction_header(grid)
 
+            grid = _split_td_continuation_date_description_grid(grid)
+            grid = _expand_td_continuation_three_col_register_grid(grid)
+
             grid = _normalize_fused_date_posted_amount_table(grid)
 
             grid = _split_fused_multicolumn_header(grid)