Update app.py

app.py

@@ -38,6 +38,11 @@ Primary goals for reconcile rate:
             — take the rightmost plausible money token from Description+Amount text for the amount.
 - Fix 20: DA3 in-table dedupe — cap consecutive identical normalized keys at 2 rows (keeps legitimate
             back-to-back identical charges; still collapses 3+ OCR stutters).
+- Fix 21: Cross-table DA3 repair for MSBILL / MICROSOFT#G… PIN rows — when Amount is junk in one table
+            but another table repeats the same authorization id with a strict amount, copy that amount
+            (then subset dedupe can drop the stray duplicate block, e.g. under Deposits).
+- Fix 22: Drop a single OCR “flat line” between pages: (continued) + DESCRIPTION + AMOUNT + many MM/DD
+            tokens (noise before a proper <table>).
 """
 
 # Patch asyncio first (before Gradio imports it) to suppress Python 3.13 cleanup noise
@@ -3153,6 +3158,8 @@ _DA3_STRICT_AMOUNT = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*\.\d{2}-?$")
 _DA3_MONEY_TOKEN = re.compile(
     r"(?<![\d,])(-?\$?\d{1,3}(?:,\d{3})*\.\d{2}|-?\$?\d{2,6}\.\d{2})(?!\d)"
 )
+_MSFT_PIN_G_REF = re.compile(r"MICROSOFT#(G\d{6,16})", re.IGNORECASE)
+_DA3_LEADING_DATE = re.compile(r"^\d{1,2}/\d{1,2}(?:/\d{2,4})?$")
 
 
 def _repair_da3_garbled_amount_cells(grid):
@@ -3381,6 +3388,122 @@ def _parse_da3_grid_from_table_html(table_html: str):
     return None
 
 
+def _strip_orphan_continued_da3_flatline(text: str) -> str:
+    """
+    Fix 22: Remove one-line OCR dumps like
+    '…(continued) DATE DESCRIPTION AMOUNT($) 04/09 … 04/11 … 239.55' that duplicate the
+    following HTML table and pollute markdown.
+    """
+    if not text or "(continued)" not in text.lower():
+        return text
+    lines = text.splitlines(keepends=True)
+    out = []
+    for line in lines:
+        core = line.rstrip("\r\n")
+        if (
+            len(core) > 80
+            and "(continued)" in core.lower()
+            and "description" in core.lower()
+            and "amount" in core.lower()
+            and len(re.findall(r"\d{2}/\d{2}", core)) >= 3
+        ):
+            continue
+        out.append(line)
+    return "".join(out)
+
+
+def _patch_da3_msft_g_ref_amounts_across_tables(text: str) -> str:
+    """
+    Fix 21: Build MICROSOFT#G… -> strict Amount from any DA3 row, then patch other DA3 rows
+    with the same id whose Amount cell is not strict currency (junk fused into Amount).
+    """
+    if not text or "<table" not in text.lower():
+        return text
+    if "microsoft#" not in text.lower():
+        return text
+
+    tbl_pat = re.compile(r"<table[^>]*>.*?</table>", re.DOTALL | re.IGNORECASE)
+
+    def _collect():
+        g_map = {}
+        ambiguous = set()
+        for m in tbl_pat.finditer(text):
+            g = _parse_da3_grid_from_table_html(m.group(0))
+            if not g:
+                continue
+            for row in g[1:]:
+                cells = [str(c or "").strip() for c in row]
+                while len(cells) < 3:
+                    cells.append("")
+                c0, c1, c2 = cells[0], cells[1], cells[2]
+                if not _DA3_LEADING_DATE.match(c0):
+                    continue
+                if not _DA3_STRICT_AMOUNT.match(c2):
+                    continue
+                blob = f"{c0} {c1} {c2}"
+                for gid in _MSFT_PIN_G_REF.findall(blob):
+                    k = gid.upper()
+                    if k in ambiguous:
+                        continue
+                    trip = (c0, c1, c2)
+                    prev = g_map.get(k)
+                    if prev is not None and prev != trip:
+                        ambiguous.add(k)
+                        g_map.pop(k, None)
+                    elif prev is None:
+                        g_map[k] = trip
+        return g_map, ambiguous
+
+    g_map, ambiguous = _collect()
+    if not g_map:
+        return text
+
+    out_chunks = []
+    last = 0
+    changed_any = False
+    for m in tbl_pat.finditer(text):
+        out_chunks.append(text[last : m.start()])
+        table_html = m.group(0)
+        grid = _parse_da3_grid_from_table_html(table_html)
+        if not grid:
+            out_chunks.append(table_html)
+            last = m.end()
+            continue
+        new_grid = [list(grid[0])]
+        changed = False
+        for row in grid[1:]:
+            cells = [str(c or "").strip() for c in row]
+            while len(cells) < 3:
+                cells.append("")
+            c0, c1, c2 = cells[0], cells[1], cells[2]
+            if not _DA3_LEADING_DATE.match(c0):
+                new_grid.append(row)
+                continue
+            if _DA3_STRICT_AMOUNT.match(c2):
+                new_grid.append(row)
+                continue
+            blob = f"{c0} {c1} {c2}"
+            ids = _MSFT_PIN_G_REF.findall(blob)
+            if len(ids) != 1:
+                new_grid.append(row)
+                continue
+            k = ids[0].upper()
+            if k in ambiguous or k not in g_map:
+                new_grid.append(row)
+                continue
+            d0, d1, d2 = g_map[k]
+            new_grid.append([d0, d1, d2])
+            changed = True
+        if changed:
+            out_chunks.append(_grid_to_html(new_grid))
+            changed_any = True
+        else:
+            out_chunks.append(table_html)
+        last = m.end()
+    out_chunks.append(text[last:])
+    return "".join(out_chunks) if changed_any else text
+
+
 def _trim_adjacent_da3_suffix_duplicating_next_table_prefix(text: str) -> str:
     """
     Remove trailing data rows from a DA3 table when they are repeated as the opening
@@ -3568,9 +3691,11 @@ def stabilize_tables_and_text(page_md: str) -> str:
 
     stabilized = "\n\n".join(out_blocks)
     stabilized = convert_plaintext_bank_sections(stabilized)
+    stabilized = _strip_orphan_continued_da3_flatline(stabilized)
     stabilized = normalize_html_tables(stabilized)
     stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
+    stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
     stabilized = _dedupe_subset_transaction_tables_html(stabilized)
     return close_unclosed_html(stabilized)
 
@@ -3700,6 +3825,7 @@ def run_ocr(uploaded_file):
                     stabilized = normalize_html_tables(stabilized)
                     stabilized = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(stabilized)
                     stabilized = _split_ucb_deposits_electronic_sections_html(stabilized)
+                    stabilized = _patch_da3_msft_g_ref_amounts_across_tables(stabilized)
                     stabilized = _dedupe_subset_transaction_tables_html(stabilized)
                 elif is_pdf and is_nfcu_doc:
                     # Navy-only additive fix: rebuild malformed summary table from
@@ -3783,6 +3909,7 @@ def run_ocr(uploaded_file):
         if all_pages and merged and not merged.startswith("Error") and "<table" in merged.lower():
             merged = _trim_adjacent_da3_suffix_duplicating_next_table_prefix(merged)
             merged = _split_ucb_deposits_electronic_sections_html(merged)
+            merged = _patch_da3_msft_g_ref_amounts_across_tables(merged)
             merged = _dedupe_subset_transaction_tables_html(merged)
         return merged