Update app.py

app.py

@@ -943,12 +943,10 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
     if the PDF has no text layer, pymupdf is unavailable, or fewer than 2
     transaction rows are found (guards against non-transaction pages).
 
-    How it works:
-      - Read all words with bounding boxes from the text layer.
-      - Group words into logical lines by clustering on y-coordinate
-        (words within 5pt of each other vertically share a line).
-      - Parse each line as a transaction row: first token = date pattern,
-        last token = amount pattern, middle tokens = description.
+    Supports multiple date formats:
+      - Numeric:    MM/DD, MM/DD/YY, MM/DD/YYYY  (Chase, TD, BoA, Citi)
+      - Month-name: Jan 16, Feb 7, Mar 05         (Capital One, Amex)
+      - ISO:        YYYY-MM-DD
     """
     try:
         import pymupdf as fitz
@@ -979,25 +977,81 @@ def _extract_textlayer_rows(pdf_path: str, page_num: int):
             line_words = sorted(lines_by_y[y], key=lambda t: t[0])
             sorted_lines.append([w for _, w in line_words])
 
-        # Date: MM/DD/YY, MM/DD/YYYY, MM/DD
-        date_re = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$")
-        # Amount: -1,234.56  $193.63  -$240.46
+        # Date patterns — numeric (MM/DD variants) or month-name (Jan 16)
+        _MONTHS = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
+        date_re = re.compile(
+            r"^\d{1,2}/\d{2}(?:/\d{2,4})?$"          # MM/DD, MM/DD/YY, MM/DD/YYYY
+            r"|^\d{4}-\d{2}-\d{2}$"                    # YYYY-MM-DD
+            r"|^" + _MONTHS + r"$",                        # "Jan", "Feb" etc (month-name date part 1)
+            re.IGNORECASE,
+        )
+        # When a month-name date appears, the next token is the day number
+        month_re = re.compile(r"^" + _MONTHS + r"$", re.IGNORECASE)
+        day_re   = re.compile(r"^\d{1,2}$")
+
+        # Amount: -1,234.56  $193.63  -$240.46  - $500.00 (with space after -)
         amount_re = re.compile(
-            r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
+            r"^-?\$?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
+            r"|^\$-?\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"
+            r"|^-\s+\$\d{1,3}(?:,\d{3})*(?:\.\d{1,4})?$"  # "- $500.00"
         )
 
         rows = []
         for line in sorted_lines:
             if len(line) < 2:
                 continue
-            if not date_re.match(line[0]):
+
+            # Detect date and find where description starts
+            date_str = None
+            desc_start = None
+
+            if date_re.match(line[0]):
+                if month_re.match(line[0]) and len(line) > 1 and day_re.match(line[1]):
+                    # Month-name date: "Jan 16" → two tokens
+                    date_str = line[0] + " " + line[1]
+                    desc_start = 2
+                else:
+                    # Single-token numeric date
+                    date_str = line[0]
+                    desc_start = 1
+            else:
                 continue
-            if not amount_re.match(line[-1]):
+
+            if desc_start is None or desc_start >= len(line):
                 continue
-            desc = " ".join(line[1:-1]).strip()
+
+            # For Capital One: Trans Date and Post Date are both present
+            # Line looks like: Jan 16 Jan 16 CAPITAL ONE MOBILE PYMT - $500.00
+            # After consuming first date, check if next tokens are also a date
+            remaining = line[desc_start:]
+            if remaining and month_re.match(remaining[0]):
+                if len(remaining) > 1 and day_re.match(remaining[1]):
+                    # Skip the post date
+                    remaining = remaining[2:]
+                elif len(remaining) > 0:
+                    remaining = remaining[1:]
+
+            if len(remaining) < 2:
+                continue
+
+            # Handle "- $500.00" split as two tokens at end
+            if (len(remaining) >= 2
+                    and remaining[-2] == "-"
+                    and remaining[-1].startswith("$")):
+                amount_candidate = remaining[-2] + " " + remaining[-1]
+                desc_tokens = remaining[:-2]
+            else:
+                amount_candidate = remaining[-1]
+                desc_tokens = remaining[:-1]
+
+            if not amount_re.match(amount_candidate):
+                continue
+
+            desc = " ".join(desc_tokens).strip()
             if not desc:
                 continue
-            rows.append({"date": line[0], "desc": desc, "amount": line[-1]})
+
+            rows.append({"date": date_str, "desc": desc, "amount": amount_candidate})
 
         # Guard: require at least 2 rows to avoid false positives on non-transaction pages
         return rows if len(rows) >= 2 else []
@@ -1085,19 +1139,37 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
             if not overlap:
                 continue
 
-            # Determine missing rows
-            to_inject = {}
+            # Determine missing rows.
+            # Case A: row exists in OCR but count is too low (duplicate dropped)
+            # Case B: row exists in text layer but is completely absent from OCR
+            #         (entire section dropped by OCR model, e.g. Capital One purchases)
+            # For Case B we build the row content from the text layer directly.
+            to_inject = {}       # key -> count of missing copies
+            to_inject_new = {}   # key -> row content for brand-new rows
+
             for key in tl_freq:
                 ocr_count = ocr_freq.get(key, 0)
                 tl_count = tl_freq[key]
-                if tl_count > ocr_count and ocr_count > 0:
-                    # Guard: only restore rows already present in OCR
-                    to_inject[key] = tl_count - ocr_count
-
-            if not to_inject:
+                if tl_count > ocr_count:
+                    missing = tl_count - ocr_count
+                    if ocr_count > 0:
+                        # Case A: restore missing duplicates (existing row template)
+                        to_inject[key] = missing
+                    else:
+                        # Case B: completely new row — build from text layer data
+                        # Find the original text-layer row for this key
+                        tl_row_data = next(
+                            (r for r in tl_rows
+                             if (_norm(r["date"]), _norm(r["desc"]), _norm(r["amount"])) == key),
+                            None
+                        )
+                        if tl_row_data:
+                            to_inject_new[key] = (missing, tl_row_data)
+
+            if not to_inject and not to_inject_new:
                 continue
 
-            # Build patched grid — inject missing rows after last occurrence
+            # Build patched grid — inject missing duplicate rows (Case A)
             new_grid = [grid[0]]
             for ri, row in enumerate(grid[1:], start=1):
                 new_grid.append(row)
@@ -1115,9 +1187,25 @@ def _patch_ocr_with_textlayer(page_md: str, pdf_path: str, page_num: int) -> str
                             new_grid.append(list(row))
                         to_inject[key] = 0
 
+            # Case B: append completely missing rows from the text layer.
+            # These are rows the OCR model dropped entirely (not just undercounted).
+            # We build new table rows using the text-layer data, placing values in
+            # the correct columns as identified from the OCR table header.
+            ncols = len(grid[0])
+            for key, (count, tl_row_data) in to_inject_new.items():
+                for _ in range(count):
+                    new_row = [""] * ncols
+                    new_row[date_col] = tl_row_data["date"]
+                    new_row[desc_col] = tl_row_data["desc"]
+                    new_row[amt_col]  = tl_row_data["amount"]
+                    new_grid.append(new_row)
+                    log.info(
+                        "page %d: injected missing row from text layer: %s %s",
+                        page_num, tl_row_data["date"], tl_row_data["desc"][:40]
+                    )
+
             new_table_html = _grid_to_html(new_grid)
             result = result.replace(table_html, new_table_html, 1)
-            log.info("page %d: injected missing duplicate row(s) from text layer", page_num)
 
         return result