SahilGoel commited on
Commit
4e4754e
·
verified ·
1 Parent(s): 50be8ee

Upload code/bank_classifier.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. code/bank_classifier.py +1213 -0
code/bank_classifier.py ADDED
@@ -0,0 +1,1213 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Bank Statement Classifier — Multi-bank parser + AI classification pipeline.
3
+
4
+ Parses PDF/CSV statements from ICICI, SBI, HDFC and generic formats.
5
+ Classifies every credit transaction as income category using:
6
+ 1. Rule engine (70-80% coverage)
7
+ 2. Recurring pattern detector (10% more)
8
+ 3. LLM fallback for remaining uncertain transactions
9
+ """
10
+ import re, json, csv, io, logging
11
+ from dataclasses import dataclass, field
12
+ from datetime import date, datetime
13
+ from decimal import Decimal
14
+ from pathlib import Path
15
+ import re as _re_module
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ BANK_PATTERNS = [
20
+ ("ICICI", ["ICICI", "icici"], "🏦"),
21
+ ("HDFC", ["HDFC", "hdfc"], "🏦"),
22
+ ("SBI", ["SBI", "State Bank", "STATE BANK"], "🏛️"),
23
+ ("Axis", ["AXIS", "axis"], "🏦"),
24
+ ("Kotak", ["KOTAK", "kotak"], "🏦"),
25
+ ("Yes Bank", ["YES BANK", "YESBANK", "yes bank"], "🏦"),
26
+ ("Federal Bank", ["FEDERAL", "FDRL", "federal"], "🏦"),
27
+ ("IDFC First", ["IDFC", "idfc"], "🏦"),
28
+ ("IndusInd", ["INDUSIND", "indusind"], "🏦"),
29
+ ("Bank of Baroda", ["BARODA", "BOB", "baroda"], "🏦"),
30
+ ("Punjab National", ["PNB", "PUNJAB NATIONAL", "pnb"], "🏦"),
31
+ ("Canara", ["CANARA", "canara"], "🏦"),
32
+ ("Union Bank", ["UNION BANK", "UNION"], "🏦"),
33
+ ("Unity SFB", ["UNITY", "unity"], "🏦"),
34
+ ]
35
+
36
+
37
+ def detect_bank(filepath: str) -> dict:
38
+ """Detect bank name from a statement file. Returns {name, icon}."""
39
+ filepath = str(filepath)
40
+ # Check filename first
41
+ fname_lower = Path(filepath).name.lower()
42
+
43
+ for bank_name, patterns, icon in BANK_PATTERNS:
44
+ for pat in patterns:
45
+ if pat.lower() in fname_lower:
46
+ return {"name": bank_name, "icon": icon}
47
+
48
+ # If not in filename, try reading the file content.
49
+ try:
50
+ suffix = Path(filepath).suffix.lower()
51
+ if suffix in ('.xls', '.xlsx'):
52
+ import pandas as pd
53
+ df = pd.read_excel(filepath, header=None)
54
+ # Check first 20 rows for bank name
55
+ for i in range(min(20, len(df))):
56
+ for j in range(min(8, len(df.columns))):
57
+ val = str(df.iloc[i, j])
58
+ for bank_name, patterns, icon in BANK_PATTERNS:
59
+ for pat in patterns:
60
+ if _re_module.search(pat, val, _re_module.IGNORECASE):
61
+ return {"name": bank_name, "icon": icon}
62
+ elif suffix == '.pdf':
63
+ import pymupdf
64
+ with pymupdf.open(filepath) as doc:
65
+ content = "\n".join(str(doc[index].get_text())
66
+ for index in range(min(3, len(doc))))
67
+ for bank_name, patterns, icon in BANK_PATTERNS:
68
+ if any(re.search(re.escape(pattern), content, re.IGNORECASE)
69
+ for pattern in patterns):
70
+ return {"name": bank_name, "icon": icon}
71
+ except Exception:
72
+ pass
73
+
74
+ return {"name": "Unknown Bank", "icon": "🏦"}
75
+ from typing import Optional
76
+ from collections import defaultdict
77
+
78
+ # Merchant DB lookup for UPI transactions
79
+ try:
80
+ from .merchant_classifier import get_merchant, extract_upi_handle
81
+ except ImportError:
82
+ from pipeline.merchant_classifier import get_merchant, extract_upi_handle
83
+
84
+
85
+ @dataclass
86
+ class RawTransaction:
87
+ """Bank-agnostic normalized transaction."""
88
+ date: date
89
+ description: str
90
+ type: str # 'credit' | 'debit'
91
+ amount: float
92
+ balance: Optional[float] = None
93
+ ref_number: Optional[str] = None
94
+
95
+ @dataclass
96
+ class ClassifiedTransaction:
97
+ """Transaction with AI classification."""
98
+ raw: RawTransaction
99
+ category: str = 'unclassified'
100
+ income_type: Optional[str] = None
101
+ confidence: float = 0.0
102
+ rationale: str = ''
103
+ is_income: bool = False
104
+ is_expense: bool = False
105
+ recurring: bool = False
106
+ counterparty: str = ''
107
+ tags: list = field(default_factory=list)
108
+
109
+ @dataclass
110
+ class ClassificationReport:
111
+ """Summary of classification results."""
112
+ total: int = 0
113
+ credits: int = 0
114
+ debits: int = 0
115
+ rule_classified: int = 0
116
+ recurring_detected: int = 0
117
+ llm_classified: int = 0
118
+ unclassified: int = 0
119
+ income_detected: int = 0
120
+ classified: list = field(default_factory=list)
121
+
122
+
123
+ # ─── Rule Engine ───────────────────────────────────────────
124
+
125
+ RULES = [
126
+ # (name, category, patterns, confidence, is_income)
127
+ ('salary', 'salary', [
128
+ r'\bSALARY\b', r'\bSAL\s', r'\bMASTERCARD\b',
129
+ r'\bPAYROLL\b', r'\bSALARIED\b',
130
+ ], 0.98, True),
131
+
132
+ ('dividend', 'dividend', [
133
+ r'\bDIVIDEND\b', r'\bDIV\s', r'\bDIVIDEND\sWARRANT\b',
134
+ r'\bACH/.*?(?:DIV|FINAL|INTERIM|INTDIV)\b', # ACH dividend payments
135
+ r'\bFINAL\s*(?:DIVIDEND|DIV)\b',
136
+ r'\bINTERIM\s*DIVIDEND\b',
137
+ # CMS dividend payments (various companies)
138
+ r'\bCMS/.*(?:DIV|LIMITED|LIMITED\s-\s\\d)', # CMS/CDSL, CMS/CESC, CMS/TECHNO
139
+ r'\bCMS/CDSL\b', r'\bCMS/CESC\b', r'\bCMS/TECHNO\b',
140
+ r'\bCMS/Deep\sIndustries\b', r'\bCMS/COMPUTER\sAGE\b',
141
+ # NEFT dividends
142
+ r'\bNEFT.*?(?:FINAL\sDIV|INTERIM\sDIV|\sDIV\s)',
143
+ r'\bBAJAJ\sHEALTHCARE\b.*\bDIV\b', r'\bVENKYS\sINDIA\b',
144
+ # Known dividend-paying companies via ACH/CMS
145
+ r'\bVARUN\sBEVERAGES\b', r'\bADANI\s?ENT', r'\bZYDUS\b',
146
+ r'\bSHANTHI\sGEARS\b', r'\bSIGACHI\b', r'\bMAITHAN\sALLOYS\b',
147
+ r'\bRPG\s?LIFE\b', r'\bHPCL\b.*\bINTDIV\b',
148
+ r'\bBRITANNIA\sINDUSTRIES\b', r'\bTATAELXSI\b',
149
+ r'\bTVS\sMOTOR\b', r'\bRAILTEL\b', r'\bREC\sLIMITED\b',
150
+ ], 0.90, True),
151
+
152
+ ('business_trust_distribution', 'other_income', [
153
+ r'\bPOWERGRID\sINFRA', r'\bINVIT\b', r'\bBUSINESS\sTRUST\b',
154
+ r'\bEMBASSY\sOFFICE\b', r'\bINDIA\sGRID\b',
155
+ ], 0.90, True),
156
+
157
+ ('interest_savings', 'interest', [
158
+ r'\bINTEREST\b', r'\bINT\sPAID\b', r'\bINT\sCR\b',
159
+ r'\bFD\sINTEREST\b', r'\bSAVINGS\sINTEREST\b', r'\bINT\.?\s?(PD|CR)\b',
160
+ ], 0.95, True),
161
+
162
+ ('rental', 'rental', [
163
+ r'\bRENT\b', r'\bHIRE\sCHARGES\b', r'\bLEASE\sRENT\b',
164
+ ], 0.85, True),
165
+
166
+ ('trading_credit', 'trading_credit', [
167
+ r'\bZERODHA', r'\bANGEL', r'\bGROWW',
168
+ r'\bUPSTOX', r'\bFYERS', r'\b5PAISA', r'\bICICI\sDIRECT',
169
+ r'\bKOTAK\sSECURITIES', r'\bMOTILAL\sOSWAL', r'\bSHAREKHAN',
170
+ r'\bBROKING', r'\bSECURITIES\sINDIA', r'\bPAYTM\sMONEY',
171
+ r'\bINDIAN\sCLEARING\sCORPORATION\b', r'\bICCL\b',
172
+ r'\bMUTUAL\sFUND.*REDEMPTION\b', r'\bCOMMON\sREDEMPTION\b',
173
+ ], 0.80, False), # NOT income — can't determine from bank statement alone
174
+
175
+ ('tax_refund', 'tax_refund', [
176
+ r'\bITD\b', r'\bINCOME\sTAX\b', r'\bIT\sREFUND\b',
177
+ r'\bCPC\b', r'\bTAX\sREFUND\b',
178
+ ], 0.95, False), # Refund, not income
179
+
180
+ ('loan_repayment', 'loan_repayment', [
181
+ r'\bLOAN\sREPAY\b', r'\bLOAN\sRETURN\b', r'\bLENDING\b',
182
+ ], 0.75, False), # Not taxable income
183
+
184
+ # ─── Additional Income Rules ───
185
+ ('salary_fdrl', 'salary', [
186
+ r'\bNEFT-FDRL.*(?:VK\sECOTRADE|CURRENT\sACCOUNT\sGENERAL)', r'\bVK\sECOTRADE\sLLP\b',
187
+ r'\bFDRL.*SALARY\b',
188
+ ], 0.95, True),
189
+
190
+ ('rental_income_known', 'rental', [
191
+ r'\bHENNYS\sWAFFLE\b', r'\bWAFFLE\sENTERPRISES\b',
192
+ r'\bsubbu526@', # Monica's tenant
193
+ r'\bRAHUL\sPANDEY\b', r'\bSAMEER\sSHAIKH\b',
194
+ r'Rent/', # ICICI BIL/INFT/...Rent/ pattern
195
+ r'\brent\b.*@[a-z]', # UPI rent payments
196
+ ], 0.88, True),
197
+
198
+ ('mf_redemption', 'trading_credit', [
199
+ r'\bMUTUAL\sFUND.*REDEMPTION\b', r'\bCOMMON\sREDEMPTION\b',
200
+ r'\bKMMF\sREDEMPTIONS\b', r'\bCANARA\sROBECO.*REDEMPTION\b',
201
+ r'\bELSS\sTAXSAVER.*INCOME\sDISTRIBUT\b',
202
+ r'\bICICI\sPRUDENTIAL.*REDEMPTION\b', # ICICI MF redemptions
203
+ ], 0.90, True),
204
+
205
+ ('interest_icici_format', 'interest', [
206
+ r':Int\.Pd:', # ICICI quarterly interest: "000501538878:Int.Pd:29-03-2025 to 29-06-2025"
207
+ ], 0.98, True),
208
+
209
+ ('family_transfer_in', 'personal_transfer', [
210
+ r'\bVINOD\sKUMAR\sGUPTA\b', # Monica's father
211
+ ], 0.70, False), # Not taxable, just flagged
212
+
213
+ # ─── Additional Expense Rules ───
214
+ ('society_maintenance', 'bills', [
215
+ r'paytm-mygate@pt', r'@PT\b.*\bMAINT', # Society maintenance
216
+ ], 0.85, False),
217
+
218
+ ('esanchala_bill', 'bills', [
219
+ r'\bE\sSANCHALA\b', r'\bESANCHALAKSOLUT\b', # Electricity/maintenance
220
+ ], 0.85, False),
221
+
222
+ ('maxbupa_insurance', 'insurance', [
223
+ r'\bMAX\sBUPA\b', r'\bMAX\sBUPA\sH\b', # Max Bupa health insurance
224
+ ], 0.92, False),
225
+
226
+ ('icici_securities_invest', 'investment', [
227
+ r'\bEBA/MFP-', # ICICI securities/insurance recurring investment
228
+ ], 0.70, False),
229
+
230
+ ('amazon_subscription', 'entertainment', [
231
+ r'\bPUR_PRIME900\b', # Amazon Prime subscription
232
+ ], 0.90, False),
233
+
234
+ ('airindia_flight', 'travel', [
235
+ r'\bairindiaexpress\b', # Air India Express flights
236
+ ], 0.80, False),
237
+
238
+ ('hospital_expense', 'medical', [
239
+ r'\bBALABHAI\sNANAVATI\b', # Hospital payments
240
+ ], 0.85, False),
241
+ ('mutual_fund_sip', 'investment', [
242
+ r'\bSIP\b', r'\bMUTUAL\sFUND\b', r'\bMUTF\b', r'\bMF\sINVEST\b',
243
+ r'\bELSS\b', r'\bNFO\b', r'\bFOLIO\b',
244
+ r'\bINDMONEY', r'\bPAYU\b.*\bMONEY\b', # No trailing \b — matches "indmoney3"
245
+ r'\bZERODHAMF\b', r'\bBSESTAR', r'\bBSE\sSTAR', # MF platforms
246
+ r'\bICCLZR@YESPAY\b', r'\bICCLZERODHA\b', r'\bZERODHA\.ICCL', # All ICCL channels = MF
247
+ ], 0.85, False),
248
+
249
+ ('staff_salary', 'staff_salary', [
250
+ r'\bRAJ\sKUMARI\b', r'\bBIHARI\sSAH\b', # Domestic staff
251
+ ], 0.85, False),
252
+
253
+ ('trading_transfer', 'trading_deposit', [
254
+ r'\bZERODHA', r'\bANGEL', r'\bGROWW',
255
+ r'\bUPSTOX', r'\bFYERS', r'\b5PAISA',
256
+ ], 0.90, False), # Money sent to trading account
257
+
258
+ ('vehicle_purchase', 'vehicle_purchase', [
259
+ r'\bASB\sAUTOMO', r'\bCAR\sDEALER\b', r'\bVEHICLE\b',
260
+ ], 0.90, False),
261
+
262
+ ('tax_payment_out', 'tax_payment', [
263
+ r'\bINCOME\sTAX\b', r'\bADVANCE\sTAX\b', r'\bSELF\sASSESSMENT\b',
264
+ r'\bCHALLAN\b', r'\bITNS\b', r'\bTDS\sPAYMENT\b',
265
+ r'\bDTAX\b', r'\bGIB/', # Tax payment patterns
266
+ ], 0.92, False),
267
+
268
+ ('toll_payment', 'travel', [ # FASTag, NHAI, toll — must be before credit_card
269
+ r'\bNHAI\b', r'\bFAST\s?TAG\b', r'\bFASTAG\b', r'\bTOLL\b',
270
+ r'\bIHMCL\b', r'\bGPTOLL\b', r'\bGP-TOLL\b', r'\bGP\.TOLL\b',
271
+ r'\bPAYTOLL\b', r'\bNETC\s?FASTAG\b',
272
+ ], 0.88, False),
273
+
274
+ ('credit_card_payment', 'credit_card', [ # conf raised to 0.95 — CRED/Unipay are unambiguous
275
+ r'\bCREDIT\sCARD\b', r'\bCC\sPAYMENT\b', r'\bUNIPAY\b.*\bCARD\b',
276
+ r'\bCARD\sPAYMENT\b', r'\bCREDITCARD\b', r'\bSIMPL\b.*\bPAY\b',
277
+ r'\bBIL/.*CREDIT\sC[A-Z]\b',
278
+ r'\bCRED\b', r'\bCRED\.', r'\bCRED\sCLUB\b', # CRED credit card payments
279
+ ], 0.90, False),
280
+
281
+ ('bill_payment', 'bills', [
282
+ r'\bBILL\b', r'\bRECHARGE\b', r'\bELECTRICITY\b', r'\bELECTRIC\b',
283
+ r'\bBROADBAND\b', r'\bWIFI\b', r'\bMOBILE\b', r'\bDTH\b',
284
+ r'\bGAS\b', r'\bWATER\b', r'\bMAINTENANCE\b',
285
+ r'\bGOOGLE\sIND', r'\bGOOGLE\b.*\bINDIA\b', # Google services
286
+ r'\bIDEALPREPA\b', # Prepaid recharge
287
+ ], 0.80, False),
288
+
289
+ ('insurance', 'insurance', [
290
+ r'\bINSURANCE\b', r'\bPREMIUM\b', r'\bLIC\b', r'\bPOLICY\b',
291
+ r'\bICICI\sPRU\b', r'\bHDFC\sLIFE\b', r'\bMAX\sLIFE\b',
292
+ r'\bTERM\sPLAN\b', r'\bHEALTH\sINSUR\b',
293
+ r'\bNivaBupa', r'\bMAX\sBUPA\b', r'\bMAX\sBUPA\sH\b', # Health insurance providers
294
+ r'BIL/ONL.*NivaBupa', r'BIL/ONL.*MAX\sBUPA', # BIL/ONL format insurance payments
295
+ ], 0.92, False),
296
+
297
+ ('grocery_delivery', 'grocery', [
298
+ r'\bBLINKIT\b', r'\bZEPTO\b', r'\bINSTAMART\b',
299
+ r'\bBIGBASKET\b', r'\bDMART\b', r'\bGROFERS\b',
300
+ ], 0.85, False),
301
+
302
+ ('loan_emi', 'loan_emi', [
303
+ r'\bEMI\b', r'\bLOAN\sREPAYMENT\b', r'\bHOME\sLOAN\b',
304
+ r'\bCAR\sLOAN\b', r'\bPERSONAL\sLOAN\b', r'\bEDUCATION\sLOAN\b',
305
+ r'\bCMS/.*SMSOTP', # Recurring CMS payments — typically loan EMI
306
+ ], 0.85, False),
307
+
308
+ ('cash_withdrawal', 'cash_withdrawal', [
309
+ r'\bATM\b', r'\bCASH\sWDL\b', r'\bCASH\sWITHDRAWAL\b',
310
+ r'\bCASH\sWDL\sRVSL\b',
311
+ ], 0.95, False),
312
+
313
+ ('gym_fitness', 'health_fitness', [
314
+ r'\bEQUANIMITY\b', r'\bINNOVANAFI\b', r'\bGYMKHANA\b',
315
+ r'\bGYM\b', r'\bFITNESS\b', r'\bKHAR\sGYM\b',
316
+ ], 0.85, False),
317
+
318
+ ('personal_transfer_out', 'personal_transfer', [
319
+ r'\bMONICA\sGOE', r'\bRUHI\sTARUN', r'\bGOELMONICA',
320
+ r'\bRUHIGOEL', r'\bTARUNKUMAR', r'\bJAI\sGUPTA',
321
+ r'\bROHIT\sAROR', r'\bBHAVISHYA', r'\bGUPTARASHI',
322
+ r'\bABHISHEK', r'\bPRIYA\sMANI', r'\bPRATEEK\sSI',
323
+ r'\bVIPUL\sCHOU', r'\bARCHITA\sBA', r'\bAARSHIN\sBA',
324
+ ], 0.80, False),
325
+
326
+ ('rent_or_property', 'rent', [
327
+ r'\bRAJ\sPHULLA\b',
328
+ ], 0.70, False),
329
+
330
+ ('paytm_merchant', 'misc_daily', [
331
+ r'PAYTM', r'@PTY', r'@PTAX', # Paytm merchant payments
332
+ ], 0.60, False),
333
+
334
+ ('rent_payment', 'rent', [
335
+ r'\bRENT\sPAY\b', r'\bRENT\sTO\b', r'\bMAINTENANCE\sCHARGE\b',
336
+ ], 0.80, False),
337
+
338
+ ('food_dining', 'food', [
339
+ r'\bSWIGGY\b', r'\bZOMATO\b', r'\bFOOD\b', r'\bRESTAURANT\b',
340
+ r'\bDOMINOS\b', r'\bMCDONALD\b', r'\bEAT\b',
341
+ ], 0.75, False),
342
+
343
+ ('shopping', 'shopping', [
344
+ r'\bAMAZON\b', r'\bFLIPKART\b', r'\bMYNTRA\b', r'\bAJIO\b',
345
+ r'\bSHOP\b', r'\bRETAIL\b', r'\bMART\b', r'\bGROCERY\b',
346
+ r'\bAPPLE\b.*\bONLIN\b', r'\bVIN/Apple\b', # Apple online store
347
+ ], 0.75, False),
348
+
349
+ ('travel', 'travel', [
350
+ r'\bUBER\b', r'\bOLA\b', r'\bRAPIDO\b', r'\bIRCTC\b',
351
+ r'\bMAKEMYTRIP\b', r'\bFLIGHT\b', r'\bAIRLINE\b', r'\bBUS\b',
352
+ ], 0.75, False),
353
+
354
+ ('entertainment', 'entertainment', [
355
+ r'\bNETFLIX\b', r'\bPRIME\b', r'\bHOTSTAR\b', r'\bSPOTIFY\b',
356
+ r'\bYOUTUBE\b', r'\bSUBSCRIPTION\b', r'\bGAME\b',
357
+ ], 0.70, False),
358
+
359
+ ('neft_transfer', 'transfer', [
360
+ r'\bNEFT-', r'\bIMPS/', r'\bRTGS',
361
+ r'\bBIL/NEFT/', # Bill payment via NEFT
362
+ ], 0.40, False), # Very low confidence — generic
363
+
364
+ ('trading_fees', 'trading_fees', [
365
+ r'\bDPCHG\b', r'\bDP\sCHGS\b', r'\bDP\sCHARGES\b',
366
+ r'\bDMC/', # Demat charges
367
+ r'\bANNUAL\sMAINTENANCE\b.*\bDP\b',
368
+ ], 0.92, False),
369
+
370
+ ('credit_card_refund', 'credit_card_refund', [
371
+ r'\bCREDIT\sCARD\b', r'\bCC\sPAYMENT\b', r'\bPAYMENT\sREVERSAL\b',
372
+ r'\bCARD\sREFUND\b',
373
+ ], 0.85, False),
374
+
375
+ ('self_transfer', 'self_transfer', [
376
+ r'\bSELF\b', r'\bOWN\sACCOUNT\b', r'\bTRANSFER\sTO\sSELF\b',
377
+ ], 0.99, False),
378
+
379
+ ('cash_deposit', 'cash_deposit', [
380
+ r'\bCASH\sDEPOSIT\b', r'\bCASH\sDEP\b', r'\bCDM\b', r'\bBY\sCASH\b',
381
+ ], 0.90, False), # Flag for review
382
+
383
+ # --- Training-data-augmented rules (NEFT/IMPS patterns) ---
384
+ ('insurance_claim', 'insurance', [
385
+ r'\bTHE\s+ORIENTAL\s+INSURANCE\b', r'\bORIENTAL\s+INS\b',
386
+ r'\bINSURANCE\s+CO\b.*\bFHP\b',
387
+ ], 0.85, True),
388
+
389
+ ('nse_settlement', 'trading_credit', [
390
+ r'\bNSE\s+CLEARING\b', r'\bMFSS\s+SETTLEMENT\b',
391
+ r'\bNSE\s+CLR\b', r'\bNSCCL\b',
392
+ ], 0.90, True),
393
+
394
+ ('gift_from_family', 'family', [
395
+ r'\bGIFT\s+TO\b', r'\bUSHA\s+GUPTA\b',
396
+ ], 0.80, True),
397
+
398
+ ('gift_in_self', 'personal_transfer', [
399
+ r'\bSAHIL\s+TARUNKUMAR\s+GOEL\b', r'\bSAHIL\s+TARU\b',
400
+ ], 0.80, False),
401
+
402
+ ('mmt_hotel', 'travel', [
403
+ r'\bMMT/IMPS.*HOUSR\b', r'\bHOUSR\s+TECH\b',
404
+ ], 0.80, False),
405
+
406
+ # --- Original catch-all rules ---
407
+ ('upi_collect', 'unclassified_credit', [
408
+ r'@[a-z]', # UPI ID pattern
409
+ ], 0.50, True), # Low confidence — needs LLM
410
+
411
+ ('neft_imps', 'unclassified_credit', [
412
+ r'\bNEFT\b', r'\bIMPS\b', r'\bRTGS\b', r'\bUPI\b',
413
+ ], 0.30, True), # Very low confidence — generic transfer
414
+ ]
415
+
416
+ # Non-income keywords that should suppress income classification
417
+ NON_INCOME_PATTERNS = [
418
+ r'\bPAYMENT\b', r'\bPURCHASE\b', r'\bFEE\b', r'\bCHARGE\b',
419
+ r'\bBILL\b', r'\bEMI\b', r'\bINSURANCE\b', r'\bPREMIUM\b',
420
+ r'\bTAX\sPAID\b', r'\bCHALLAN\b',
421
+ ]
422
+
423
+
424
+ _pipeline = None
425
+
426
+ def _get_pipeline():
427
+ global _pipeline
428
+ if _pipeline is None:
429
+ from .classifier import ClassificationPipeline
430
+ from .classifier.stages import MerchantDBStage, UPIHeuristicStage, DescriptionRuleStage, RegexRuleStage, LLMFallbackStage, CatchAllStage
431
+ _pipeline = ClassificationPipeline([
432
+ MerchantDBStage(),
433
+ UPIHeuristicStage(),
434
+ DescriptionRuleStage(),
435
+ RegexRuleStage(),
436
+ LLMFallbackStage(),
437
+ CatchAllStage(),
438
+ ])
439
+ return _pipeline
440
+
441
+ def _derive_tags(category: str, counterparty: str, description: str) -> list:
442
+ """Derive faceted tags from category, counterparty, and description.
443
+
444
+ Tags enable multi-dimensional filtering: a Zomato transaction gets
445
+ ["food", "delivery", "zomato"] in addition to its primary category.
446
+ """
447
+ tags = []
448
+ desc_lower = description.lower()
449
+
450
+ # Category is always the primary tag
451
+ tags.append(category)
452
+
453
+ # Channel tag (how the payment was made)
454
+ if desc_lower.startswith('upi/') or '@' in desc_lower:
455
+ tags.append('upi')
456
+ elif desc_lower.startswith('imps'):
457
+ tags.append('imps')
458
+ elif desc_lower.startswith('neft'):
459
+ tags.append('neft')
460
+ elif desc_lower.startswith('rtgs'):
461
+ tags.append('rtgs')
462
+ elif desc_lower.startswith('nach'):
463
+ tags.append('nach')
464
+ elif 'atm' in desc_lower:
465
+ tags.append('atm')
466
+ elif 'card' in desc_lower or 'pos' in desc_lower:
467
+ tags.append('card')
468
+
469
+ # Merchant tag (normalized counterparty)
470
+ if counterparty:
471
+ merchant_tag = re.sub(r'[^a-z0-9]', '', counterparty.lower())[:20]
472
+ if merchant_tag and merchant_tag != category:
473
+ tags.append(merchant_tag)
474
+
475
+ # Purpose tags (semantic facets)
476
+ purpose_map = {
477
+ 'food': ['delivery', 'restaurant'],
478
+ 'grocery': ['essential'],
479
+ 'medical': ['healthcare'],
480
+ 'travel': ['transport'],
481
+ 'shopping': ['online'],
482
+ 'entertainment': ['subscription'],
483
+ 'investment': ['sip', 'mutual_fund'],
484
+ 'trading_deposit': ['stock'],
485
+ 'credit_card': ['bill_payment'],
486
+ 'insurance': ['premium'],
487
+ 'education': ['tuition'],
488
+ 'loan_emi': ['loan'],
489
+ }
490
+ if category in purpose_map:
491
+ tags.extend(purpose_map[category])
492
+
493
+ # Income tag
494
+ from pipeline.training_schema import INCOME_CATEGORIES
495
+ if category in INCOME_CATEGORIES:
496
+ tags.append('income')
497
+
498
+ # Deduplicate while preserving order
499
+ seen = set()
500
+ return [t for t in tags if not (t in seen or seen.add(t))]
501
+
502
+
503
+ CARD_ISSUER_PATTERNS: list[tuple] = [
504
+ (re.compile(r"(?i)ICICI\\s*BANK\\s*CREDIT\\s*CA|icici\\s*bank\\s*card"), "ICICI Credit Card"),
505
+ (re.compile(r"(?i)HDFC\\s*BANK\\s*CREDIT|hdfc\\s*bank\\s*card"), "HDFC Credit Card"),
506
+ (re.compile(r"(?i)SBI\\s*CARD|sbicard|sbi\\s*credit\\s*card"), "SBI Credit Card"),
507
+ (re.compile(r"(?i)AXIS\\s*BANK\\s*CREDIT|axis\\s*bank\\s*card|AXIS.*?CARD"), "Axis Credit Card"),
508
+ (re.compile(r"(?i)AMEX|AMERICAN\\s*EXPRESS"), "Amex"),
509
+ (re.compile(r"(?i)KOTAK\\s*MAHINDRA.*CARD|kotak.*credit"), "Kotak Credit Card"),
510
+ (re.compile(r"(?i)RBL\\s*CARD|rbl.*credit"), "RBL Credit Card"),
511
+ (re.compile(r"(?i)YES\\s*BANK.*CARD|yes.*credit.*card"), "Yes Bank Credit Card"),
512
+ (re.compile(r"(?i)INDUSIND.*CREDIT.*CARD|indusind.*card"), "IndusInd Credit Card"),
513
+ (re.compile(r"(?i)STANDARD\\s*CHARTERED.*CARD|SCB.*CREDIT"), "StanChart Credit Card"),
514
+ (re.compile(r"(?i)HSBC.*CREDIT.*CARD"), "HSBC Credit Card"),
515
+ (re.compile(r"(?i)CITI.*CREDIT.*CARD|CITIBANK.*CARD"), "Citi Credit Card"),
516
+ (re.compile(r"(?i)AU\\s*BANK.*CARD|AU.*CREDIT"), "AU Credit Card"),
517
+ (re.compile(r"(?i)IDFC.*CREDIT|IDFC.*CARD"), "IDFC Credit Card"),
518
+ (re.compile(r"(?i)BOB\\s*CARD|BANK\\s*OF\\s*BARODA.*CARD|bobcard|onecard"), "BOB/OneCard"),
519
+ ]
520
+
521
+
522
+ def _extract_card_issuer(description: str) -> str:
523
+ """Extract credit card issuer name from transaction description."""
524
+ for pattern, issuer in CARD_ISSUER_PATTERNS:
525
+ if pattern.search(description):
526
+ return issuer
527
+ return ""
528
+
529
+
530
+ def classify_with_rules(
531
+ txn: RawTransaction,
532
+ *,
533
+ learn_merchants: bool = True,
534
+ ) -> Optional[ClassifiedTransaction]:
535
+ """Apply rule engine. Income rules match credits, expense rules match debits.
536
+
537
+ Order: Merchant DB lookup → regex rules → catch-all.
538
+ """
539
+ result = _get_pipeline().classify(txn, learn=learn_merchants)
540
+ if result is None:
541
+ return None
542
+
543
+ counterparty = result.counterparty
544
+
545
+ # Extract credit card issuer if not already set
546
+ if result.category == 'credit_card' and not counterparty:
547
+ counterparty = _extract_card_issuer(txn.description)
548
+
549
+ return ClassifiedTransaction(
550
+ raw=txn,
551
+ category=result.category,
552
+ confidence=result.confidence,
553
+ is_income=result.is_income,
554
+ is_expense=result.is_expense,
555
+ counterparty=counterparty,
556
+ rationale=result.rationale,
557
+ income_type=result.income_type or None,
558
+ tags=list(getattr(result, 'tags', [])) or _derive_tags(result.category, result.counterparty, txn.description),
559
+ )
560
+
561
+
562
+ # ─── Recurring Detector ────────────────────────────────────
563
+
564
+ class RecurringDetector:
565
+ """Detects recurring income patterns across transactions."""
566
+
567
+ def detect(self, transactions: list[ClassifiedTransaction]) -> list[ClassifiedTransaction]:
568
+ """Find recurring patterns in unclassified credits."""
569
+ # Group by counterparty (sender extracted from description)
570
+ groups = defaultdict(list)
571
+ for ctxn in transactions:
572
+ if ctxn.category == 'unclassified' and ctxn.raw.type == 'credit':
573
+ cp = self._extract_counterparty(ctxn.raw.description)
574
+ groups[cp].append(ctxn)
575
+
576
+ for cp, group in groups.items():
577
+ if len(group) < 2:
578
+ continue
579
+
580
+ dates = sorted(tx.raw.date for tx in group)
581
+ amounts = [tx.raw.amount for tx in group]
582
+
583
+ # Check for monthly pattern
584
+ if self._is_monthly(dates) and self._amount_stable(amounts):
585
+ for tx in group:
586
+ tx.category = 'rental' if 'rent' in tx.raw.description.lower() else 'recurring_income'
587
+ tx.confidence = 0.82
588
+ tx.is_income = True
589
+ tx.rationale = f'Recurring monthly: {cp} — {len(group)} occurrences'
590
+ tx.recurring = True
591
+ tx.counterparty = cp
592
+
593
+ return transactions
594
+
595
+ def _extract_counterparty(self, desc: str) -> str:
596
+ """Extract sender name from transaction description."""
597
+ # Credit card issuer detection
598
+ for pattern, issuer in CARD_ISSUER_PATTERNS:
599
+ if pattern.search(desc):
600
+ return issuer
601
+
602
+ # NEFT: NEFT-SENDER_NAME-BANK
603
+ m = re.search(r'NEFT[-\s]+([A-Za-z0-9\s]+?)[-\s]+', desc)
604
+ if m: return m.group(1).strip()[:40]
605
+ # IMPS: IMPS/SENDER/...
606
+ m = re.search(r'IMPS[-\s/]+([A-Za-z0-9\s]+?)[-\s/]', desc)
607
+ if m: return m.group(1).strip()[:40]
608
+ # UPI: sender@bank
609
+ m = re.search(r'([a-zA-Z0-9_.]+@[a-zA-Z]+)', desc)
610
+ if m: return m.group(1)
611
+ # Fallback: first word
612
+ return desc.split()[0] if desc else 'unknown'
613
+
614
+ def _is_monthly(self, dates: list[date], tolerance: int = 5) -> bool:
615
+ """Check if dates are approximately monthly."""
616
+ if len(dates) < 2:
617
+ return False
618
+ for i in range(1, len(dates)):
619
+ delta = abs((dates[i] - dates[i-1]).days)
620
+ if not (25 <= delta <= 35):
621
+ return False
622
+ return True
623
+
624
+ def _amount_stable(self, amounts: list[float], tolerance: float = 0.05) -> bool:
625
+ """Check if amounts are within tolerance percentage of each other."""
626
+ if not amounts:
627
+ return False
628
+ avg = sum(amounts) / len(amounts)
629
+ return all(abs(a - avg) / avg <= tolerance for a in amounts if avg > 0)
630
+
631
+
632
+ # ─── LLM Classifier (stub) ─────────────────────────────────
633
+
634
+ class LLMClassifier:
635
+ """Classifies uncertain transactions using the fine-tuned Qwen 0.5B model."""
636
+
637
+ def __init__(self):
638
+ self._local = None
639
+
640
+ def _get_model(self):
641
+ if self._local is None:
642
+ try:
643
+ from pipeline.llm_classifier import get_llm_classifier
644
+ self._local = get_llm_classifier()
645
+ if not self._local.available:
646
+ logger.warning("Qwen model not available — transactions will need manual review")
647
+ except Exception as exc:
648
+ logger.warning("Failed to import LLM classifier: %s", exc)
649
+ self._local = False
650
+ return self._local if self._local and self._local is not False else None
651
+
652
+ def classify_batch(
653
+ self,
654
+ transactions: list[ClassifiedTransaction],
655
+ ) -> list[ClassifiedTransaction]:
656
+ """Re-classify uncertain transactions using the local Qwen model."""
657
+ model = self._get_model()
658
+ if model is None:
659
+ for tx in transactions:
660
+ if tx.category == "unclassified":
661
+ tx.rationale = "Needs manual review (LLM not available)"
662
+ return transactions
663
+
664
+ # Only classify unclassified or low-confidence transactions
665
+ uncertain = [
666
+ tx for tx in transactions
667
+ if tx.category == "unclassified" or tx.confidence < 0.70
668
+ ]
669
+
670
+ if not uncertain:
671
+ return transactions
672
+
673
+ for tx in uncertain:
674
+ result = model.classify(
675
+ description=tx.raw.description,
676
+ txn_type=tx.raw.type,
677
+ )
678
+ if result and result.confidence >= 0.50:
679
+ tx.category = result.category
680
+ tx.confidence = result.confidence
681
+ tx.counterparty = result.company_name or tx.counterparty
682
+ tx.rationale = result.rationale
683
+ tx.is_income = result.is_income
684
+ else:
685
+ tx.rationale = "LLM uncertain — needs manual review"
686
+
687
+ return transactions
688
+
689
+
690
+ # ─── Pipeline Orchestrator ─────────────────────────────────
691
+
692
+ def classify_bank_statement(filepath: str) -> ClassificationReport:
693
+ """
694
+ Full classification pipeline:
695
+ 1. Parse statement → RawTransaction[]
696
+ 2. Rule classifier
697
+ 3. Recurring detector
698
+ 4. LLM fallback
699
+ Returns ClassificationReport with stats and classified transactions.
700
+ """
701
+ report = ClassificationReport()
702
+
703
+ # Step 1: Parse
704
+ raw_txns = _parse_statement(filepath)
705
+ report.total = len(raw_txns)
706
+ report.credits = sum(1 for t in raw_txns if t.type == 'credit')
707
+ report.debits = sum(1 for t in raw_txns if t.type == 'debit')
708
+
709
+ # Step 2: Rules
710
+ classified = []
711
+ for raw in raw_txns:
712
+ result = classify_with_rules(raw)
713
+ if result:
714
+ classified.append(result)
715
+ else:
716
+ classified.append(ClassifiedTransaction(raw=raw))
717
+
718
+ report.rule_classified = sum(1 for c in classified if c.category != 'unclassified')
719
+
720
+ # Step 3: Recurring
721
+ classified = RecurringDetector().detect(classified)
722
+ report.recurring_detected = sum(1 for c in classified if c.recurring)
723
+
724
+ # Step 4: LLM
725
+ uncertain = [c for c in classified if c.category == 'unclassified' and c.raw.type == 'credit']
726
+ if uncertain:
727
+ classified = LLMClassifier().classify_batch(classified)
728
+
729
+ report.classified = classified
730
+ report.unclassified = sum(1 for c in classified if c.category == 'unclassified')
731
+ report.income_detected = sum(1 for c in classified if c.is_income)
732
+
733
+ return report
734
+
735
+
736
+ # ─── Statement Parsers ─────────────────────────────────────
737
+
738
+ def _parse_statement(filepath: str) -> list[RawTransaction]:
739
+ """Route to appropriate parser based on file extension and content."""
740
+ path = Path(filepath)
741
+ ext = path.suffix.lower()
742
+
743
+ if ext == '.csv':
744
+ return _parse_csv(path)
745
+ elif ext == '.pdf':
746
+ return _parse_pdf(path)
747
+ elif ext in ('.xls', '.xlsx'):
748
+ return _parse_icici_excel(str(path))
749
+ else:
750
+ # Try CSV first, then PDF
751
+ try:
752
+ return _parse_csv(path)
753
+ except:
754
+ return _parse_pdf(path)
755
+
756
+ def _parse_icici_excel(filepath: str) -> list[RawTransaction]:
757
+ """Parse ICICI Bank XLS/XLSX statement (JasperReports format).
758
+
759
+ Auto-detects the column layout — some files have an extra NaN column at index 0.
760
+ """
761
+ import pandas as pd
762
+ df = pd.read_excel(filepath, header=None)
763
+
764
+ # Detect if there's an extra NaN column at index 0 (Monica/user format)
765
+ col0_is_nan = True
766
+ for i in range(min(15, len(df))):
767
+ if pd.notna(df.iloc[i, 0]):
768
+ col0_is_nan = False
769
+ break
770
+
771
+ col_offset = 1 if col0_is_nan else 0
772
+
773
+ # Find header row
774
+ data_start = 8 # default
775
+ for i in range(20):
776
+ v = str(df.iloc[i, col_offset]) if pd.notna(df.iloc[i, col_offset]) else ''
777
+ if v == 'S No.':
778
+ data_start = i + 1
779
+ break
780
+
781
+ transactions = []
782
+ for i in range(data_start, len(df)):
783
+ # Skip rows without a valid S.No. (continuation lines, footers)
784
+ sno = str(df.iloc[i, col_offset]) if pd.notna(df.iloc[i, col_offset]) else ''
785
+ if not sno.isdigit():
786
+ continue
787
+
788
+ desc_col = col_offset + 4
789
+ withdrawal_col = col_offset + 5
790
+ deposit_col = col_offset + 6
791
+ desc = str(df.iloc[i, desc_col]) if pd.notna(df.iloc[i, desc_col]) else ''
792
+ withdrawal = df.iloc[i, withdrawal_col] if pd.notna(df.iloc[i, withdrawal_col]) else 0
793
+ deposit = df.iloc[i, deposit_col] if pd.notna(df.iloc[i, deposit_col]) else 0
794
+
795
+ if not desc or desc == 'nan':
796
+ continue
797
+
798
+ try:
799
+ w = float(withdrawal) if withdrawal and str(withdrawal) != 'nan' else 0.0
800
+ d = float(deposit) if deposit and str(deposit) != 'nan' else 0.0
801
+ except (TypeError, ValueError):
802
+ continue
803
+ txn_type = 'credit' if d > 0 else 'debit'
804
+ amount = d if d > 0 else w
805
+
806
+ date_col = col_offset + 2
807
+ date_str = str(df.iloc[i, date_col]) if pd.notna(df.iloc[i, date_col]) else ''
808
+ try:
809
+ txn_date = pd.to_datetime(date_str, dayfirst=True).date()
810
+ except:
811
+ txn_date = date.today()
812
+
813
+ transactions.append(RawTransaction(
814
+ date=txn_date, description=desc.strip(),
815
+ type=txn_type, amount=amount
816
+ ))
817
+
818
+ return sorted(transactions, key=lambda t: t.date)
819
+
820
+
821
+ def _parse_credit_card(filepath: str) -> list[RawTransaction]:
822
+ """Parse a credit card statement (CSV/PDF/XLSX).
823
+
824
+ Credit card CSVs typically have: Date, Description, Amount (all debits).
825
+ Merging these with bank statements fills expense gaps — the bank only shows
826
+ one bulk payment to the card company, but the card statement has every purchase.
827
+ """
828
+ path = Path(filepath)
829
+ ext = path.suffix.lower()
830
+
831
+ if ext == '.csv':
832
+ return _parse_credit_card_csv(path)
833
+ elif ext in ('.xls', '.xlsx'):
834
+ return _parse_credit_card_excel(filepath)
835
+ else:
836
+ # Try CSV first
837
+ try:
838
+ return _parse_credit_card_csv(path)
839
+ except:
840
+ return []
841
+
842
+
843
+ def _parse_credit_card_csv(path: Path) -> list[RawTransaction]:
844
+ """Parse credit card CSV. Auto-detects columns."""
845
+ transactions = []
846
+ with open(path, encoding='utf-8-sig') as f:
847
+ reader = csv.DictReader(f)
848
+ if not reader.fieldnames:
849
+ return transactions
850
+
851
+ cols = [c.lower().strip() for c in reader.fieldnames]
852
+
853
+ # Detect date column
854
+ date_col = next((c for c in cols if 'date' in c and 'post' not in c), cols[0] if len(cols) > 0 else None)
855
+ # Detect description column
856
+ desc_col = next((c for c in cols if c in ('description','narration','particulars','transaction details','details')), None)
857
+ if not desc_col:
858
+ desc_col = next((c for c in cols if 'desc' in c), cols[1] if len(cols) > 1 else None)
859
+ # Detect amount column
860
+ amt_col = next((c for c in cols if c in ('amount','transaction amount','inr','rs.')), None)
861
+ if not amt_col:
862
+ amt_col = next((c for c in cols if 'amount' in c), cols[2] if len(cols) > 2 else None)
863
+
864
+ for row in reader:
865
+ try:
866
+ desc = str(row.get(desc_col, '')).strip()
867
+ amt_str = str(row.get(amt_col, '0')).replace(',', '').replace('₹', '').replace('Rs.', '').strip()
868
+ amt = abs(float(amt_str)) if amt_str else 0
869
+ if not desc or amt <= 0:
870
+ continue
871
+
872
+ date_str = str(row.get(date_col, ''))
873
+ try:
874
+ from dateutil.parser import parse as dateparse
875
+ txn_date = dateparse(date_str).date()
876
+ except:
877
+ txn_date = date.today()
878
+
879
+ transactions.append(RawTransaction(date=txn_date, description=desc, type='debit', amount=amt))
880
+ except (ValueError, KeyError):
881
+ continue
882
+
883
+ return sorted(transactions, key=lambda t: t.date)
884
+
885
+
886
+ def _parse_credit_card_excel(filepath: str) -> list[RawTransaction]:
887
+ """Parse credit card XLS/XLSX. Reads first sheet, auto-detects columns."""
888
+ import pandas as pd
889
+ try:
890
+ df = pd.read_excel(filepath)
891
+ except:
892
+ return []
893
+
894
+ if df.empty:
895
+ return []
896
+
897
+ # Normalize column names
898
+ df.columns = [str(c).lower().strip() for c in df.columns]
899
+ cols = list(df.columns)
900
+
901
+ date_col = next((c for c in cols if 'date' in c and 'post' not in c), cols[0] if cols else None)
902
+ desc_col = next((c for c in cols if c in ('description','narration','particulars')), cols[1] if len(cols) > 1 else None)
903
+ amt_col = next((c for c in cols if 'amount' in c or c in ('inr','rs.')), cols[2] if len(cols) > 2 else None)
904
+
905
+ if not all([date_col, desc_col, amt_col]):
906
+ return []
907
+
908
+ transactions = []
909
+ for _, row in df.iterrows():
910
+ try:
911
+ desc = str(row[desc_col]).strip()
912
+ amt = abs(float(str(row[amt_col]).replace(',', '').replace('₹', ''))) if pd.notna(row[amt_col]) else 0
913
+ if not desc or amt <= 0 or desc == 'nan':
914
+ continue
915
+ txn_date = pd.to_datetime(row[date_col]).date() if pd.notna(row[date_col]) else date.today()
916
+ transactions.append(RawTransaction(date=txn_date, description=desc, type='debit', amount=amt))
917
+ except (ValueError, KeyError):
918
+ continue
919
+
920
+ return sorted(transactions, key=lambda t: t.date)
921
+
922
+
923
+ def _parse_csv(path: Path) -> list[RawTransaction]:
924
+ """Parse a CSV bank statement. Columns: Date, Description, Debit, Credit, Balance."""
925
+ transactions = []
926
+ with open(path) as f:
927
+ reader = csv.DictReader(f)
928
+ for row in reader:
929
+ try:
930
+ txn_date = _parse_date(row.get('Date', row.get('date', '')))
931
+ desc = row.get('Description', row.get('description', row.get('Narration', '')))
932
+ debit = float(str(row.get('Debit', row.get('debit', '0')).replace(',', '')))
933
+ credit = float(str(row.get('Credit', row.get('credit', '0')).replace(',', '')))
934
+ balance = row.get('Balance', row.get('balance', ''))
935
+ bal = float(str(balance).replace(',', '')) if balance else None
936
+
937
+ if debit > 0:
938
+ transactions.append(RawTransaction(date=txn_date, description=desc,
939
+ type='debit', amount=debit, balance=bal))
940
+ elif credit > 0:
941
+ transactions.append(RawTransaction(date=txn_date, description=desc,
942
+ type='credit', amount=credit, balance=bal))
943
+ except (ValueError, KeyError):
944
+ continue
945
+
946
+ return sorted(transactions, key=lambda t: t.date)
947
+
948
+ def _parse_pdf(path: Path) -> list[RawTransaction]:
949
+ """Parse PDF bank statement using pymupdf. Handles ICICI, SBI, HDFC formats."""
950
+ try:
951
+ import pymupdf
952
+ except ImportError:
953
+ raise ImportError("pymupdf required for PDF parsing. Run: pip install pymupdf")
954
+
955
+ with pymupdf.open(str(path)) as doc:
956
+ text = "\n".join(str(page.get_text()) for page in doc)
957
+
958
+ # Indie exports lose blank debit/credit cells in plain-text order.
959
+ # Parse visual amount columns while PDF coordinates are available.
960
+ if _is_indie_icici_text(text):
961
+ return _parse_indie_icici_pdf(doc)
962
+
963
+ # Detect bank and parse accordingly
964
+ if 'ICICI Bank' in text:
965
+ return _parse_icici_text(text)
966
+ elif 'State Bank of India' in text or 'SBI' in text:
967
+ return _parse_sbi_text(text)
968
+ elif 'HDFC Bank' in text:
969
+ return _parse_hdfc_text(text)
970
+ else:
971
+ return _parse_generic_text(text)
972
+
973
+
974
+ _INDIE_DATE_RE = re.compile(r'^\d{2}\.\d{2}\.\d{4}$')
975
+ _INDIE_AMOUNT_RE = re.compile(r'^-?[\d,]+\.\d{2}$')
976
+ _INDIE_SNO_RE = re.compile(r'^\d{1,4}$')
977
+
978
+
979
+ def _is_indie_icici_text(text: str) -> bool:
980
+ upper = text.upper()
981
+ return ('STATEMENT OF TRANSACTIONS IN SAVING ACCOUNT' in upper
982
+ or ('ICICI BANK LIMITED' in upper
983
+ and 'WITHDRAWAL' in upper
984
+ and 'DEPOSIT' in upper
985
+ and bool(re.search(r'\d{2}\.\d{2}\.\d{4}', text))))
986
+
987
+
988
+ def _parse_indie_icici_text(text: str) -> list[RawTransaction]:
989
+ """Parse synthetic Indie-style text only when direction is explicit.
990
+
991
+ Plain extraction omits blank withdrawal/deposit cells, so a positive amount
992
+ is not evidence of direction. Real PDFs must use `_parse_indie_icici_pdf`.
993
+ """
994
+ lines = [line.strip() for line in text.splitlines()]
995
+ transactions = []
996
+ i = 0
997
+ while i + 1 < len(lines):
998
+ if not (_INDIE_SNO_RE.fullmatch(lines[i])
999
+ and _INDIE_DATE_RE.fullmatch(lines[i + 1])):
1000
+ i += 1
1001
+ continue
1002
+
1003
+ txn_date = _parse_date(lines[i + 1])
1004
+ description_lines = []
1005
+ j = i + 2
1006
+ while j < len(lines) and not _INDIE_AMOUNT_RE.fullmatch(lines[j]):
1007
+ if (_INDIE_SNO_RE.fullmatch(lines[j]) and j + 1 < len(lines)
1008
+ and _INDIE_DATE_RE.fullmatch(lines[j + 1])):
1009
+ break
1010
+ if lines[j]:
1011
+ description_lines.append(lines[j])
1012
+ j += 1
1013
+
1014
+ amounts = []
1015
+ while j < len(lines) and _INDIE_AMOUNT_RE.fullmatch(lines[j]):
1016
+ amounts.append(float(lines[j].replace(',', '')))
1017
+ j += 1
1018
+
1019
+ description = ' '.join(description_lines).strip()
1020
+ has_credit = bool(re.search(r'\bCREDIT\b', description, re.IGNORECASE))
1021
+ has_debit = bool(re.search(r'\bDEBIT\b', description, re.IGNORECASE))
1022
+ if len(amounts) >= 2 and description and has_credit != has_debit:
1023
+ transactions.append(RawTransaction(
1024
+ date=txn_date,
1025
+ description=description,
1026
+ type='credit' if has_credit else 'debit',
1027
+ amount=amounts[0],
1028
+ balance=amounts[-1],
1029
+ ))
1030
+ i = max(j, i + 1)
1031
+
1032
+ return sorted(transactions, key=lambda txn: txn.date)
1033
+
1034
+
1035
+ def _page_text_lines(page) -> list[tuple[float, float, float, float, str]]:
1036
+ """Return visual PDF lines as `(x0, y0, x1, y1, text)` tuples."""
1037
+ result = []
1038
+ for block in page.get_text('dict').get('blocks', []):
1039
+ for line in block.get('lines', []):
1040
+ text = ''.join(span.get('text', '') for span in line.get('spans', [])).strip()
1041
+ if text:
1042
+ x0, y0, x1, y1 = line['bbox']
1043
+ result.append((x0, y0, x1, y1, text))
1044
+ return sorted(result, key=lambda item: (item[1], item[0]))
1045
+
1046
+
1047
+ def _parse_indie_icici_pdf(doc) -> list[RawTransaction]:
1048
+ """Parse Indie exports using withdrawal/deposit/balance x-coordinates."""
1049
+ transactions = []
1050
+ for page in doc:
1051
+ lines = _page_text_lines(page)
1052
+ date_rows = [line for line in lines if _INDIE_DATE_RE.fullmatch(line[4])]
1053
+ width = float(page.rect.width)
1054
+
1055
+ for index, date_line in enumerate(date_rows):
1056
+ row_top = date_line[1] - 1.0
1057
+ row_bottom = (date_rows[index + 1][1] - 1.0
1058
+ if index + 1 < len(date_rows) else float(page.rect.height))
1059
+ row = [line for line in lines if row_top <= line[1] < row_bottom]
1060
+
1061
+ withdrawal = []
1062
+ deposit = []
1063
+ balances = []
1064
+ for x0, _y0, x1, _y1, value in row:
1065
+ if not _INDIE_AMOUNT_RE.fullmatch(value) or x0 < width * 0.65:
1066
+ continue
1067
+ parsed = float(value.replace(',', ''))
1068
+ if x1 <= width * 0.78:
1069
+ withdrawal.append(parsed)
1070
+ elif x1 <= width * 0.89:
1071
+ deposit.append(parsed)
1072
+ else:
1073
+ balances.append(parsed)
1074
+
1075
+ # Direction comes exclusively from the populated visual amount column.
1076
+ has_withdrawal = len(withdrawal) == 1 and withdrawal[0] != 0
1077
+ has_deposit = len(deposit) == 1 and deposit[0] != 0
1078
+ if has_withdrawal == has_deposit or len(balances) != 1:
1079
+ continue
1080
+
1081
+ description_parts = [
1082
+ value for x0, _y0, x1, _y1, value in row
1083
+ if width * 0.30 <= x0 and x1 < width * 0.67
1084
+ and not _INDIE_AMOUNT_RE.fullmatch(value)
1085
+ ]
1086
+ description = ' '.join(description_parts).strip()
1087
+ if not description:
1088
+ continue
1089
+
1090
+ transactions.append(RawTransaction(
1091
+ date=_parse_date(date_line[4]),
1092
+ description=description,
1093
+ type='debit' if has_withdrawal else 'credit',
1094
+ amount=withdrawal[0] if has_withdrawal else deposit[0],
1095
+ balance=balances[0],
1096
+ ))
1097
+
1098
+ return sorted(transactions, key=lambda txn: txn.date)
1099
+
1100
+
1101
+ def _parse_icici_text(text: str) -> list[RawTransaction]:
1102
+ """Parse ICICI Bank statement text."""
1103
+ transactions = []
1104
+ # ICICI format: Date | Description | Cheque No | Debit | Credit | Balance
1105
+ lines = text.split('\n')
1106
+ in_txn_section = False
1107
+
1108
+ for line in lines:
1109
+ line = line.strip()
1110
+ if not line:
1111
+ continue
1112
+
1113
+ # Detect transaction section start
1114
+ if re.search(r'Date\s+Description\s+.*(?:Debit|Credit)', line):
1115
+ in_txn_section = True
1116
+ continue
1117
+
1118
+ if in_txn_section:
1119
+ # Try to match: DD/MM/YYYY Description ... Amount Amount Amount
1120
+ match = re.match(r'(\d{2}/\d{2}/\d{4})\s+(.+?)\s+([\d,]+\.?\d*)\s*$', line)
1121
+ if not match:
1122
+ match = re.match(r'(\d{2}/\d{2}/\d{4})\s+(.+?)\s+([\d,]+\.?\d*)\s+([\d,]+\.?\d*)', line)
1123
+
1124
+ if match:
1125
+ txn_date = _parse_date(match.group(1))
1126
+ desc = match.group(2).strip()
1127
+ amounts = [float(g.replace(',', '')) for g in match.groups()[2:] if g]
1128
+
1129
+ if len(amounts) >= 2:
1130
+ if amounts[-2] > 0: # Debit
1131
+ transactions.append(RawTransaction(
1132
+ date=txn_date, description=desc, type='debit',
1133
+ amount=amounts[-2], balance=amounts[-1] if len(amounts) > 2 else None))
1134
+ elif amounts[-1] > 0: # Credit
1135
+ transactions.append(RawTransaction(
1136
+ date=txn_date, description=desc, type='credit',
1137
+ amount=amounts[-1], balance=amounts[-2] if len(amounts) > 2 else None))
1138
+ elif len(amounts) == 1:
1139
+ txn_type = 'credit' if 'CR' in desc.upper() or 'credit' in desc.lower() else 'debit'
1140
+ transactions.append(RawTransaction(
1141
+ date=txn_date, description=desc, type=txn_type, amount=amounts[0]))
1142
+
1143
+ return sorted(transactions, key=lambda t: t.date)
1144
+
1145
+ def _parse_sbi_text(text: str) -> list[RawTransaction]:
1146
+ """Parse SBI statement. Falls back to generic parser."""
1147
+ return _parse_generic_text(text)
1148
+
1149
+ def _parse_hdfc_text(text: str) -> list[RawTransaction]:
1150
+ """Parse HDFC statement. Falls back to generic parser."""
1151
+ return _parse_generic_text(text)
1152
+
1153
+ def _parse_generic_text(text: str) -> list[RawTransaction]:
1154
+ """Generic parser — looks for date + amount patterns."""
1155
+ transactions = []
1156
+ for line in text.split('\n'):
1157
+ line = line.strip()
1158
+ # Look for DD/MM/YYYY or DD-MM-YYYY followed by amount
1159
+ match = re.search(r'(\d{2}[/-]\d{2}[/-]\d{4}).*?([\d,]+\.?\d{2})', line)
1160
+ if match:
1161
+ try:
1162
+ txn_date = _parse_date(match.group(1))
1163
+ amount = float(match.group(2).replace(',', ''))
1164
+ except ValueError:
1165
+ continue
1166
+ desc = line[:match.start(2)].strip()
1167
+ txn_type = 'credit' if ('CR' in line.upper() or amount > 10000) else 'debit'
1168
+ transactions.append(RawTransaction(date=txn_date, description=desc, type=txn_type, amount=amount))
1169
+ return sorted(transactions, key=lambda t: t.date)
1170
+
1171
+
1172
+ def _parse_date(s: str) -> date:
1173
+ """Parse date from various formats."""
1174
+ s = s.strip()
1175
+ for fmt in ['%d/%m/%Y', '%d-%m-%Y', '%d.%m.%Y', '%Y-%m-%d', '%d/%m/%y', '%m/%d/%Y']:
1176
+ try:
1177
+ return datetime.strptime(s, fmt).date()
1178
+ except ValueError:
1179
+ continue
1180
+ raise ValueError(f"Cannot parse date: {s}")
1181
+
1182
+
1183
+ # ─── CLI ───────────────────────────────────────────────────
1184
+
1185
+ if __name__ == '__main__':
1186
+ import sys
1187
+ if len(sys.argv) < 2:
1188
+ print("Usage: python bank_classifier.py <statement.pdf|csv>")
1189
+ print(" Classifies bank transactions into income categories.")
1190
+ sys.exit(1)
1191
+
1192
+ report = classify_bank_statement(sys.argv[1])
1193
+
1194
+ print(f"Total transactions: {report.total}")
1195
+ print(f"Credits: {report.credits} | Debits: {report.debits}")
1196
+ print(f"Rule-classified: {report.rule_classified}")
1197
+ print(f"Recurring detected: {report.recurring_detected}")
1198
+ print(f"Unclassified: {report.unclassified}")
1199
+ print(f"Income detected: {report.income_detected}")
1200
+ print()
1201
+
1202
+ # Show income transactions
1203
+ print("=== Income Transactions ===")
1204
+ for txn in report.classified:
1205
+ if txn.is_income:
1206
+ print(f" {txn.raw.date} | ₹{txn.raw.amount:>10,.2f} | {txn.category:25s} | {txn.confidence:.0%} | {txn.raw.description[:60]}")
1207
+
1208
+ # Show unclassified credits
1209
+ unclassified = [t for t in report.classified if t.category == 'unclassified' and t.raw.type == 'credit']
1210
+ if unclassified:
1211
+ print(f"\n=== Unclassified Credits ({len(unclassified)}) — Needs Review ===")
1212
+ for txn in unclassified[:20]:
1213
+ print(f" {txn.raw.date} | ₹{txn.raw.amount:>10,.2f} | {txn.raw.description[:80]}")