Parser: multi-issuer support (HDFC/Axis/IDFC/Kotak), forex/fee fixes, city-split, brand aliases

app/merchants.py

@@ -114,6 +114,38 @@ MERCHANT_MAP: Dict[str, Tuple[str, Optional[str]]] = {
     "insurance": ("insurance", None),
     "premium": ("insurance", None),
     "rent": ("rent", None),
+    # ride-hailing / transport
+    "uber": ("transport", "uber"),
+    "rapido": ("transport", "rapido"),
+    "olacabs": ("transport", "ola"),
+    "ola cabs": ("transport", "ola"),
+    # developer tools / online services (kept neutral so rewards aren't distorted)
+    "github": ("online_shopping", "github"),
+    "notion": ("online_shopping", "notion"),
+    "colab": ("online_shopping", None),
+    "google": ("online_shopping", "google"),
+    "figma": ("online_shopping", None),
+    "vercel": ("online_shopping", None),
+    "digitalocean": ("online_shopping", None),
+    # test prep / education (correct category — GMAT/MBA prep is education spend)
+    "gmatclub": ("education", None),
+    "gmat": ("education", None),
+    "targettestprep": ("education", None),
+    "target test prep": ("education", None),
+    "rocketblocks": ("education", None),
+    "coursera": ("education", None),
+    "udemy": ("education", None),
+    "leetcode": ("education", None),
+    "kaplan": ("education", None),
+    "upgrad": ("education", None),
+    # more Indian brands
+    "poshvine": ("general", None),
+    "urbanclap": ("general", "urban_company"),
+    "urban company": ("general", "urban_company"),
+    "lenskart": ("online_shopping", "lenskart"),
+    "mamaearth": ("online_shopping", "mamaearth"),
+    "decathlon": ("apparel", "decathlon"),
+    "fashnear": ("online_shopping", "meesho"),
     # wallet
     "paytm wallet": ("wallet_load", None),
     "wallet load": ("wallet_load", None),

app/statement_parser.py

@@ -43,9 +43,12 @@ _DATE_TOKEN_RE = re.compile(
     r"(\d{1,2}[/\-.]\d{1,2}[/\-.]\d{2,4}|\d{1,2}[ \-]\w{3}[ \-]\d{2,4}|\d{4}-\d{2}-\d{2})"
 )
 # Rows whose description is really a payment/credit, not a purchase.
+# No trailing \b: HDFC glues the next word onto "PAYMENT" ("CREDIT CARD PAYMENTNet
+# Banking"), which a trailing word-boundary would miss, letting a bill payment be
+# counted as spend.
 _PAYMENT_RE = re.compile(
     r"\b(payment received|credit card payment|card payment|neft|imps|rtgs|upi[\s/-]*payment|"
-    r"autopay|auto pay|bbps|nach|e-?mandate|refund|reversal|cashback received)\b",
+    r"autopay|auto pay|bbps|nach|e-?mandate|refund|reversal|cashback received)",
     re.IGNORECASE,
 )
 
@@ -53,17 +56,82 @@ _PAYMENT_RE = re.compile(
 # not purchases the user chooses a card for, so they're excluded from the analysis.
 _CHARGES_RE = re.compile(
     r"(\bigst\b|\bcgst\b|\bsgst\b|\bgst\b|finance charge|interest charge|\binterest\b|"
-    r"late fee|membership fee|annual fee|joining fee|renewal fee|cash advance fee|"
-    r"over\s?limit fee|fuel surcharge|surcharge)",
+    r"late fee|late payment|membership fee|annual fee|joining fee|renewal fee|cash advance fee|"
+    r"over\s?limit fee|fuel surcharge|surcharge|"
+    r"forex markup|fx markup|markup fee|currency conversion|cross[\s-]?currency|"
+    r"reward point|\bfee reversal\b|\bemi (?:principal|interest)\b)",
     re.IGNORECASE,
 )
 
 
+# Foreign-currency codes on international lines, and helpers to drop forex-conversion
+# detail sub-lines and to render a clean merchant name from a raw descriptor.
+_CCY_RE = re.compile(r"\b(usd|eur|gbp|aed|sgd|jpy|aud|cad|chf|hkd|thb|myr|sar|qar|cny)\b", re.IGNORECASE)
+_FOREX_DETAIL_RE = re.compile(r"^(convert|conversion|fx|forex|foreign\s*currency|markup|intl\s*txn)\b", re.IGNORECASE)
+_CCY_AMT_RE = re.compile(_CCY_RE.pattern + r"\s*[\d.,]+", re.IGNORECASE)
+_PREFIX_RE = re.compile(r"^\s*(pos|vps|ecom(m)?|imps|neft|rtgs|upi|ach|nach|intl|int'?l)[\s/*:.\-]+", re.IGNORECASE)
+
+
+def _is_forex_detail_line(desc: str) -> bool:
+    d = (desc or "").strip()
+    return bool(_FOREX_DETAIL_RE.match(d) and _CCY_RE.search(d))
+
+
+def _title(s: str) -> str:
+    return re.sub(r"\bPaypal\b", "PayPal", s.title())
+
+
+# HDFC glues the city onto the merchant with no space ("Entertainmenthyderabad",
+# "Kurlamumbai"). Split a trailing known city off, longest names first.
+_CITIES = tuple(sorted((
+    "navimumbai", "newdelhi", "bengaluru", "bangalore", "hyderabad", "ahmedabad",
+    "coimbatore", "visakhapatnam", "bhubaneswar", "chandigarh", "gurugram", "gurgaon",
+    "mumbai", "delhi", "chennai", "kolkata", "pune", "noida", "jaipur", "lucknow",
+    "kochi", "indore", "nagpur", "surat", "thane", "vadodara", "bhopal", "patna",
+    "ludhiana", "agra", "nashik", "faridabad", "ghaziabad", "rajkot", "meerut",
+    "amritsar", "mysuru", "mysore", "goa", "kanpur", "varanasi", "guwahati",
+), key=len, reverse=True))
+
+
+def _split_glued_city(s: str) -> str:
+    low = s.lower()
+    for c in _CITIES:
+        if low.endswith(c) and len(s) > len(c) + 2 and s[-len(c) - 1] not in " ,":
+            return s[:-len(c)] + " " + s[-len(c):]
+    return s
+
+
+def _clean_merchant(raw: str) -> str:
+    original = (raw or "").strip()
+    strip = lambda x: re.sub(r"^[\s*.,\-]+|[\s*.,\-]+$", "", x).strip()
+    if re.search(r"paypal", original, re.IGNORECASE):
+        m = re.search(r"paypal\s*\*?\s*([a-z][a-z0-9 &._-]{1,24})", original, re.IGNORECASE)
+        sub = ""
+        if m and m.group(1):
+            sub = strip(re.sub(r"[\d.,]+", "", re.sub(r"\b(convert|conversion|usd|eur|gbp)\b", "", m.group(1), flags=re.IGNORECASE)))
+        sub = sub.upper() if len(sub) <= 4 else _title(sub)
+        return f"PayPal · {sub}" if len(sub) > 1 else "PayPal"
+    s = _PREFIX_RE.sub("", original)
+    s = _CCY_AMT_RE.sub(" ", s)
+    s = re.sub(r"\b(convert|conversion)\b", " ", s, flags=re.IGNORECASE)
+    s = re.sub(r"#\S*", " ", s)
+    s = re.sub(r"\d[\d.,]{3,}", " ", s)
+    s = re.sub(r"\*+", " ", s)
+    s = strip(re.sub(r"\s{2,}", " ", s))
+    if len(s) < 2:
+        return original[:40] or "Transaction"
+    return _title(_split_glued_city(s))[:40]
+
+
 def _parse_date(s: str) -> Optional[str]:
     """Parse a date that may carry a trailing timestamp (e.g. '17/04/2026 22:46:14')."""
     s = (s or "").strip()
     if not s:
         return None
+    # Axis prints the year with a leading apostrophe ("01 Jan '26") - drop it so
+    # strptime's %y matches, and collapse the double space it leaves behind.
+    s = re.sub(r"'\s*(\d)", r"\1", s)
+    s = re.sub(r"\s{2,}", " ", s)
     # pull just the date token if there's a time or extra text alongside it
     tok = _DATE_TOKEN_RE.search(s)
     candidates = [s]
@@ -117,7 +185,7 @@ def _normalize_row(date_s: str, desc: str, amount: float) -> Dict:
     return {
         "date": _parse_date(date_s) or date_s,
         "description": desc.strip()[:120],
-        "merchant": resolved["matched_merchant"] or desc.strip()[:40],
+        "merchant": resolved["matched_merchant"] or _clean_merchant(desc),
         "category": resolved["category"],
         "brand_key": resolved["brand_key"],
         "amount": round(abs(amount), 2),
@@ -186,6 +254,8 @@ def parse_csv(content: bytes) -> List[Dict]:
             direction = r[ki] if 0 <= ki < len(r) else ""
             if _is_credit(direction, desc):
                 continue
+            if _is_forex_detail_line(desc):     # drop forex conversion sub-lines ("Convert USD 324.50")
+                continue
             out.append(_normalize_row(iso, desc, amount))
         return out  # header found: trust it (even if every row was a credit/payment)
 
@@ -206,12 +276,14 @@ def parse_csv(content: bytes) -> List[Dict]:
         desc = max(others, key=len) if others else " ".join(r)
         if _is_credit("", desc):
             continue
+        if _is_forex_detail_line(desc):
+            continue
         out.append(_normalize_row(_parse_date(date_field), desc, amount))
     return out
 
 
 # a date anywhere on a line: dd/mm/yyyy, dd-mm-yy, dd Mon yy, dd-MON-yy, dd/Mon/yyyy
-_PDF_DATE_RE = re.compile(r"(\d{1,2}[/\-. ](?:\d{1,2}|[A-Za-z]{3,9})[/\-. ]\d{2,4})")
+_PDF_DATE_RE = re.compile(r"(\d{1,2}[/\-. ](?:\d{1,2}|[A-Za-z]{3,9})[/\-. ]'?\d{2,4})")
 _SIGNED_AMT_RE = re.compile(r"(-?\d[\d,]*\.\d{2})")
 # lines that are clearly not transactions (summary / headers / footers)
 _PDF_STOP = (
@@ -221,6 +293,43 @@ _PDF_STOP = (
 )
 _PDF_START = ("transaction details", "your transactions", "transaction date", "date transaction")
 
+# Amex prints month-first dates with no year on the row ("June 03"); the year comes
+# from the statement period. Support that with a separate matcher + year injection.
+_MONTHS = {m: i for i, m in enumerate(
+    ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"], 1)}
+_MONTH_DAY_RE = re.compile(
+    r"\b((?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+\d{1,2})(?!\d)", re.I)
+
+
+def _parse_month_day(txt: str, year: int) -> Optional[str]:
+    m = re.match(r"([a-z]{3})[a-z]*\.?\s+(\d{1,2})", (txt or "").strip(), re.I)
+    if not m:
+        return None
+    mo = _MONTHS.get(m.group(1).lower())
+    day = int(m.group(2))
+    if not mo or not (1 <= day <= 31):
+        return None
+    return f"{year:04d}-{mo:02d}-{day:02d}"
+
+
+# Descriptor noise seen on Kotak (and others): payment-gateway prefixes, embedded URLs,
+# brand domains, and the glued "SpendsArea" MCC-group column printed before the amount.
+_DOMAIN_RE = re.compile(r"\b([a-z][\w-]*)\.(?:com|in|org|net|io|co|edu|gov|sg|us|uk|app)\w*", re.I)
+_AGG_PREFIX_RE = re.compile(r"\b(raz|pyu|payu|ccbill|billdesk|pinelabs)\s*\*\s*", re.I)
+_SPENDS_AREA_RE = re.compile(
+    r"\s+(services|education|automotive|apparel(?:\s*&\s*accessories)?|"
+    r"transport\s*&?\s*freight|freight|quasi\s*cash|financial\s*services|"
+    r"government\s*services|professional\s*services|utilities|telecom|insurance)\s*$", re.I)
+
+
+def _strip_descriptor_noise(s: str) -> str:
+    s = re.sub(r"https?[:/]*\S*", " ", s, flags=re.I)   # URLs, incl. glued "httpsgmat.."
+    s = re.sub(r"\([^)]*\)", " ", s)                    # "(*ConverttoEMI)" markers
+    s = re.sub(r"\s*\([^)]*$", " ", s)                  # a marker truncated at a line wrap "(*Conv"
+    s = _AGG_PREFIX_RE.sub(" ", s)                      # "RAZ*RAPIDO" -> "RAPIDO"
+    s = _DOMAIN_RE.sub(r"\1", s)                        # "github.com" -> "github"
+    return re.sub(r"\s{2,}", " ", s).strip()
+
 
 class _PdfRow:
     pass
@@ -230,6 +339,10 @@ def _parse_pdf_lines(lines: List[str], gated: bool = True) -> List[Dict]:
     out: List[Dict] = []
     started = not gated          # ungated fallback: parse from the top
     pending: List[str] = []      # buffered wrapped merchant-name lines (IDFC style)
+    # Year for month-first dates that omit it (Amex "June 03"): take the latest 4-digit
+    # year printed anywhere on the statement, else the current year.
+    _yrs = re.findall(r"\b(20\d{2})\b", "\n".join(lines))
+    default_year = int(max(_yrs)) if _yrs else datetime.now().year
 
     for raw in lines:
         line = (raw or "").strip()
@@ -243,10 +356,12 @@ def _parse_pdf_lines(lines: List[str], gated: bool = True) -> List[Dict]:
             continue
 
         dm = _PDF_DATE_RE.search(line)
+        md = None if dm else _MONTH_DAY_RE.search(line)      # Amex month-first fallback
+        date_txt = dm.group(1) if dm else (md.group(1) if md else None)
         amts = list(_SIGNED_AMT_RE.finditer(line))
 
-        if dm and amts:
-            iso = _parse_date(dm.group(1))
+        if date_txt and amts:
+            iso = _parse_date(date_txt) if dm else _parse_month_day(date_txt, default_year)
             if not iso:                      # date-like but not a real date -> buffer text
                 if re.search(r"[A-Za-z]", line):
                     pending = (pending + [line])[-2:]
@@ -259,26 +374,58 @@ def _parse_pdf_lines(lines: List[str], gated: bool = True) -> List[Dict]:
             if amount == 0:                  # FX-only / zero rows
                 pending = []
                 continue
-            desc = line[:amt_m.start()].replace(dm.group(1), " ")
+            desc = line[:amt_m.start()].replace(date_txt, " ")
+            desc = desc.replace("₹", " ")                 # Axis prints "₹ 980.00"
+            desc = re.sub(r"\b(?:inr|rs)\.?\b", " ", desc, flags=re.I)  # RBL "INR 1,899.00" prefix
+            desc = _strip_descriptor_noise(desc)          # Kotak URLs / gateway prefix / EMI markers / domains
+            desc = re.sub(r"^[\s|]*\d{1,2}:\d{2}(?::\d{2})?\s+", " ", desc)  # HDFC leading time "22:16"
+            desc = re.sub(r"\s\+\s*\d{1,3}\b", " ", desc)  # HDFC "+ 15" reward-point marker
             desc = re.sub(r"\b\d{6,}\b", " ", desc)       # strip long reference numbers
+            desc = re.sub(r"#\w+", " ", desc)             # Axis "#HJ1S1I6CZYEUZ9" transaction refs
             desc = re.sub(r"\b0\.00\b", " ", desc)        # strip the FX (international) 0.00 column
             desc = re.sub(r"\b[DC]R\b", " ", desc, flags=re.I)
             desc = re.sub(r"(?<![A-Za-z])[rR](?=\d)", " ", desc)  # ₹ rendered as 'r'
-            desc = re.sub(r"\s{2,}", " ", desc).strip(" ,|")
-            if not re.search(r"[A-Za-z]", desc):          # no inline name -> use wrapped buffer
-                desc = " ".join(pending).strip(" ,|")
+            desc = re.sub(r"\s+[CD]\s*$", " ", desc)      # HDFC trailing debit/credit column letter
+            desc = _SPENDS_AREA_RE.sub(" ", desc)         # Kotak "... Services"/"... Automotive" MCC column
+            desc = re.sub(r"\s+in\s*$", " ", desc, flags=re.I)          # trailing India country code
+            desc = re.sub(r"\b(\w+)(\s+\1\b)+", r"\1", desc, flags=re.I)  # collapse repeated words ("Gmatclub Gmatclub")
+            desc = re.sub(r"\s{2,}", " ", desc).strip(" ,|+")
+            # A real merchant name is one that survives stripping the issuer's EMI-convert
+            # marker and any "USD 324.50" forex fragment. IDFC prints an international
+            # purchase as "<merchant wrapped above>\n<date> Convert USD 324.50 <INR>", so
+            # the amount line itself has no merchant - recover it from the wrapped buffer.
+            core = re.sub(r"\b(convert|conversion|fx|forex)\b", " ", desc, flags=re.I)
+            core = _CCY_AMT_RE.sub(" ", core)
+            core = _CCY_RE.sub(" ", core)
+            core = re.sub(r"[\d.,]+", " ", core).strip(" ,|*")
+            if len(core) < 2:                             # inline text is only a forex marker
+                desc = " ".join(pending).strip(" ,|") or desc
             pending = []
             if is_credit:
                 continue
+            # Re-run the credit/charge check on the RESOLVED description: on wrapped
+            # layouts (IDFC) the merchant text comes from `pending`, so the raw `line`
+            # check above never sees words like "Interest charges" / "Forex Markup Fee".
+            if _is_credit("", desc) or _CHARGES_RE.search(desc):
+                continue
             # ungated fallback: require a real merchant name so summary/total lines
             # (which have no description) aren't mistaken for transactions
             if not gated and not re.search(r"[A-Za-z]", desc):
                 continue
+            # last resort: an international row whose merchant never resolved - label it
+            # rather than showing the raw "Convert USD 324.50" forex fragment
+            if _is_forex_detail_line(desc):
+                desc = "International transaction"
             out.append(_normalize_row(iso, desc or "Transaction", amount))
-        elif not dm and re.search(r"[A-Za-z]", line) and len(line) <= 60 and not any(k in low for k in _PDF_STOP):
-            pending = (pending + [line])[-2:]   # candidate wrapped merchant name
         else:
-            pending = []
+            # Candidate wrapped merchant name (no amount on this line). Some issuers
+            # (RBL) repeat the date on the continuation line, so strip any leading date
+            # before buffering it for the amount row that follows.
+            nm = (line.replace(date_txt, " ") if date_txt else line).strip(" ,|")
+            if re.search(r"[A-Za-z]", nm) and len(nm) <= 60 and not any(k in low for k in _PDF_STOP):
+                pending = (pending + [nm])[-2:]
+            else:
+                pending = []
     return out