Update ml_utils.py

ml_utils.py

@@ -1,137 +1,232 @@
-# ml_utils.py improved
+# ml_utils.py, adding ensemble for URL scam detection
 
 import re
 import pickle
 import logging
+import numpy as np
 from typing import Dict, List, Tuple
 from urllib.parse import urlparse
 from pathlib import Path
 
+import pandas as pd
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier
 from sklearn.pipeline import Pipeline
-import pandas as pd
+from sklearn.model_selection import cross_val_score
+from xgboost import XGBClassifier
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
+# ── URL feature engineering ───────────────────────────────────────────────────
+
+def extract_url_features(url: str) -> Dict:
+    """Extract structured numerical features from a URL string."""
+    try:
+        raw = url
+        if not url.startswith(('http://', 'https://')):
+            url = 'http://' + url
+        p = urlparse(url)
+        domain = p.netloc.lower()
+        path = p.path.lower()
+        full = url.lower()
+
+        SECURITY_KW = ['login', 'verify', 'update', 'confirm', 'secure',
+                       'account', 'kyc', 'banking', 'signin', 'validation']
+        BRAND_KW    = ['hdfc', 'sbi', 'icici', 'paytm', 'amazon', 'google',
+                       'microsoft', 'paypal', 'netflix', 'airtel', 'jio',
+                       'phonepe', 'razorpay', 'flipkart', 'swiggy', 'zomato',
+                       'axis', 'kotak', 'uidai', 'npci', 'irctc']
+        SUSP_TLDS   = ['.tk', '.ml', '.ga', '.cf', '.gq', '.xyz', '.top',
+                       '.pw', '.click', '.site', '.co', '.in']
+        FREE_TLDS   = ['.tk', '.ml', '.ga', '.cf', '.gq']
+
+        return {
+            'length':                int(len(raw)),
+            'dot_count':             int(raw.count('.')),
+            'hyphen_count':          int(raw.count('-')),
+            'slash_count':           int(raw.count('/')),
+            'digit_count':           int(sum(c.isdigit() for c in domain)),
+            'is_https':              int(url.startswith('https://')),
+            'is_ip':                 int(bool(re.search(r'\d+\.\d+\.\d+\.\d+', domain))),
+            'subdomain_depth':       int(max(len(domain.split('.')) - 2, 0)),
+            'has_security_kw':       int(any(w in full for w in SECURITY_KW)),
+            'has_brand_kw':          int(any(w in full for w in BRAND_KW)),
+            'suspicious_tld':        int(any(domain.endswith(t) for t in SUSP_TLDS)),
+            'has_free_tld':          int(any(domain.endswith(t) for t in FREE_TLDS)),
+            'path_length':           int(len(p.path)),
+            'has_numbers_in_domain': int(bool(re.search(r'\d', domain.split('.')[0]))),
+            'hyphen_in_domain':      int('-' in domain),
+            'multi_hyphens':         int(domain.count('-') >= 2),
+            'at_sign':               int('@' in url),
+            'double_slash_redirect': int('//' in p.path),
+            'query_length':          int(len(p.query)),
+            'brand_plus_hyphen':     int(any(w in domain and '-' in domain for w in BRAND_KW)),
+            'security_kw_in_path':   int(any(w in path for w in SECURITY_KW)),
+        }
+    except Exception:
+        return {k: 0 for k in [
+            'length', 'dot_count', 'hyphen_count', 'slash_count', 'digit_count',
+            'is_https', 'is_ip', 'subdomain_depth', 'has_security_kw', 'has_brand_kw',
+            'suspicious_tld', 'has_free_tld', 'path_length', 'has_numbers_in_domain',
+            'hyphen_in_domain', 'multi_hyphens', 'at_sign', 'double_slash_redirect',
+            'query_length', 'brand_plus_hyphen', 'security_kw_in_path',
+        ]}
+
+
 class ScamDetectionService:
     def __init__(self):
-        logger.info("Loading spam detection model...")
+        logger.info("Loading models...")
 
-        model_path = Path("spam_model.pkl")
+        text_path = Path("spam_model.pkl")
+        url_path  = Path("url_ensemble.pkl")
 
-        if model_path.exists():
-            with open(model_path, 'rb') as f:
-                self.model = pickle.load(f)
-            logger.info("Model loaded from disk.")
+        if text_path.exists():
+            with open(text_path, 'rb') as f:
+                self.text_model = pickle.load(f)
+            logger.info("Text model loaded.")
         else:
-            logger.info("Training new model...")
-            self.train_model()
+            logger.info("Training text model...")
+            self.train_text_model()
+
+        if url_path.exists():
+            with open(url_path, 'rb') as f:
+                self.url_ensemble = pickle.load(f)
+            logger.info("URL ensemble loaded.")
+        else:
+            logger.info("Training URL ensemble...")
+            self.train_url_ensemble()
 
         self.suspicious_shorteners = [
             'bit.ly', 'tinyurl.com', 'short.link', 'tiny.cc', 'ow.ly',
             'goo.gl', 't.co', 'rb.gy', 'is.gd', 'v.gd', 'cutt.ly'
         ]
 
-        # Each tuple: (regex_pattern, risk_score, short_reason, detailed_reason)
+        # (regex, risk_score, short_label, detailed_explanation)
         self.social_engineering_patterns: List[Tuple[str, float, str, str]] = [
             # OTP / credential harvesting
-            (r'\bshare.{0,20}otp\b',                              0.90, "OTP request",         "Asks you to share an OTP — no legitimate org ever does this."),
-            (r'\bconfirm.{0,20}(otp|pin|password)\b',            0.85, "Credential request",   "Requests you confirm an OTP, PIN, or password — a classic phishing tactic."),
-            (r'\b(last\s*4|last\s*four).{0,20}(digit|card)\b',   0.85, "Card digit request",   "Asks for the last 4 digits of your card — used to build up to full card theft."),
-            (r'\b(card\s*number|cvv|expiry)\b',                   0.85, "Card detail request",  "Requests card number, CVV, or expiry — your bank will never ask for these over SMS."),
-            (r'\botp\b',                                          0.45, "OTP mention",           "Mentions OTP — context suggests a transaction or verification prompt."),
-
+            (r'\bshare.{0,20}otp\b',                              0.90, "OTP request",
+             "Asks you to share an OTP -- no legitimate org ever does this."),
+            (r'\bconfirm.{0,20}(otp|pin|password)\b',            0.85, "Credential request",
+             "Requests you confirm an OTP, PIN, or password -- classic phishing."),
+            (r'\b(last\s*4|last\s*four).{0,20}(digit|card)\b',   0.85, "Card digit request",
+             "Asks for last 4 digits of your card -- used to build to full card theft."),
+            (r'\b(card\s*number|cvv|expiry)\b',                   0.85, "Card detail request",
+             "Requests card number, CVV, or expiry -- your bank will never ask over SMS."),
+            (r'\botp\b',                                          0.45, "OTP mention",
+             "Mentions OTP -- context suggests a transaction or verification prompt."),
             # Bank / govt impersonation
             (r'\b(bank|rbi|sbi|hdfc|icici|axis|kotak|pnb|boi).{0,30}(suspend|block|close|deactivat|restrict)\b',
-                                                                  0.88, "Bank account threat",  "Claims your bank account is being suspended or blocked — banks do this via official mail, not SMS links."),
+             0.88, "Bank account threat",
+             "Claims your bank account is being suspended -- banks use official mail, not SMS links."),
             (r'\b(fraud\s*prevention|fraud\s*team|fraud\s*department|fraud\s*monitoring)\b',
-                                                                  0.75, "Fraud team impersonation", "Impersonates a bank's fraud team to create panic and urgency."),
+             0.75, "Fraud team impersonation",
+             "Impersonates a bank fraud team to create panic and urgency."),
             (r'\b(kyc|know\s*your\s*customer).{0,20}(update|expire|pending|complet|verif)\b',
-                                                                  0.88, "KYC scam",             "KYC update requests via SMS are almost always fraudulent. Use only your bank's official app."),
+             0.88, "KYC scam",
+             "KYC update requests via SMS are almost always fraudulent."),
             (r'\baadhaar.{0,30}(link|update|verify|expire|deactivat|biometric)\b',
-                                                                  0.85, "Aadhaar scam",         "UIDAI does not send Aadhaar deactivation or verification requests via SMS."),
+             0.85, "Aadhaar scam",
+             "UIDAI does not send Aadhaar deactivation or verification requests via SMS."),
             (r'\b(pan\s*card|pan).{0,30}(flag|block|verify|link|update)\b',
-                                                                  0.82, "PAN card scam",        "PAN verification is done only through the income tax portal — not SMS links."),
+             0.82, "PAN card scam",
+             "PAN verification is done only through the income tax portal -- not SMS links."),
             (r'\b(rbi|reserve\s*bank).{0,30}(notice|compliance|regulat|review|audit)\b',
-                                                                  0.85, "RBI impersonation",    "The RBI does not contact individuals directly via SMS for compliance or audits."),
+             0.85, "RBI impersonation",
+             "The RBI does not contact individuals directly via SMS for compliance or audits."),
             (r'\b(income\s*tax|it\s*department|tds).{0,30}(refund|notice|verif|confirm|scrutin)\b',
-                                                                  0.85, "Tax dept scam",        "Income tax refunds and notices come through the official e-Filing portal, not SMS."),
+             0.85, "Tax dept scam",
+             "Income tax refunds and notices come through e-Filing portal, not SMS."),
             (r'\b(gst|epfo|uan|pf\s*deposit).{0,30}(verif|update|link|suspend|block)\b',
-                                                                  0.82, "Govt portal impersonation", "Legitimate EPFO/GST communications don't ask for verification through SMS links."),
-
+             0.82, "Govt portal impersonation",
+             "Legitimate EPFO/GST communications don't ask for verification via SMS links."),
             # Telecom
             (r'\b(sim|mobile).{0,30}(deactivat|block|suspend|port).{0,20}(kyc|verif|update)\b',
-                                                                  0.85, "SIM KYC scam",         "TRAI and telecom operators don't deactivate SIMs via SMS verification links."),
+             0.85, "SIM KYC scam",
+             "TRAI and telecom operators don't deactivate SIMs via SMS verification links."),
             (r'\b(airtel|jio|vi|vodafone|bsnl).{0,30}(block|suspend|deactivat|kyc|port)\b',
-                                                                  0.82, "Telecom impersonation", "Telecom providers handle SIM issues at stores or via their official apps — not SMS links."),
-
+             0.82, "Telecom impersonation",
+             "Telecom providers handle SIM issues at stores or official apps -- not SMS links."),
             # Digital arrest / legal threat
             (r'\b(cbi|cybercrime|enforcement\s*directorate|ed|police|court).{0,40}(case|notice|filed|investigation|arrest|prosecution)\b',
-                                                                  0.90, "Law enforcement impersonation", "CBI/ED/Police do not initiate legal proceedings via SMS. This is a 'digital arrest' scam pattern."),
+             0.90, "Law enforcement impersonation",
+             "CBI/ED/Police do not initiate legal proceedings via SMS. This is a 'digital arrest' scam."),
             (r'\b(section\s*420|fema|pmla|ipc|money\s*laundering).{0,40}(invest|notice|case|compli)\b',
-                                                                  0.88, "Legal threat scam",    "Citing specific legal sections over SMS to create fear is a known fraud tactic."),
-            (r'\b(legal\s*notice|warrant|fir)\b',                 0.80, "Legal threat",         "Legitimate legal notices arrive through official postal or court channels, not SMS."),
-
+             0.88, "Legal threat scam",
+             "Citing specific legal sections over SMS to create fear is a known fraud tactic."),
+            (r'\b(legal\s*notice|warrant|fir)\b',
+             0.80, "Legal threat",
+             "Legitimate legal notices arrive through official postal or court channels, not SMS."),
             # Account suspension / urgency
             (r'\b(account|service|upi|wallet).{0,20}(suspend|block|terminat|deactivat|restrict)\b',
-                                                                  0.75, "Account suspension threat", "Suspension threats via SMS are a pressure tactic to make you act without thinking."),
+             0.75, "Account suspension threat",
+             "Suspension threats via SMS are pressure tactics to make you act without thinking."),
             (r'\b(immediate|immediately|urgent|urgently).{0,30}(action|call|contact|verify|confirm)\b',
-                                                                  0.70, "Urgency pressure",     "Manufactured urgency is the #1 social engineering tactic — designed to bypass rational thinking."),
-            (r'\bwithin\s*(1|2|24|30|48|72)\s*hours?\b',         0.65, "Time pressure",        "Artificial deadlines are used to pressure you into acting before you can verify the request."),
-
+             0.70, "Urgency pressure",
+             "Manufactured urgency is the #1 social engineering tactic -- bypasses rational thinking."),
+            (r'\bwithin\s*(1|2|24|30|48|72)\s*hours?\b',
+             0.65, "Time pressure",
+             "Artificial deadlines pressure you into acting before you can verify."),
             # Fake helpline
             (r'\b(call|contact).{0,20}(helpline|support|team|officer|number).{0,30}\d{8,12}\b',
-                                                                  0.75, "Fake helpline",        "Scammers publish fake helpline numbers — always call numbers from the official company website."),
-
+             0.75, "Fake helpline",
+             "Scammers publish fake helpline numbers -- always call from the official website."),
             # Prize / lottery
             (r'\b(won|win|winner|winning).{0,30}(prize|lottery|lucky|reward|cash|gift|iphone|samsung)\b',
-                                                                  0.90, "Lottery/prize scam",   "You cannot win a lottery you didn't enter. These are designed to get your personal details."),
+             0.90, "Lottery/prize scam",
+             "You cannot win a lottery you didn't enter. Designed to get your personal details."),
             (r'\bcongratulations.{0,50}(won|selected|chosen|winner|shortlist)\b',
-                                                                  0.90, "Prize scam",           "Unsolicited congratulations messages are almost universally fraudulent."),
+             0.90, "Prize scam",
+             "Unsolicited congratulations messages are almost universally fraudulent."),
             (r'\bclaim.{0,20}(prize|reward|cash|gift|money|amount)\b',
-                                                                  0.85, "Claim prize prompt",   "Asking you to 'claim' a prize you weren't expecting is a classic advance fee setup."),
-
+             0.85, "Claim prize prompt",
+             "Asking you to 'claim' a prize you weren't expecting is a classic advance fee setup."),
             # Job fraud
             (r'\b(work\s*from\s*home|earn.{0,10}per\s*day|daily\s*earning|part\s*time\s*job).{0,40}(register|fee|pay|deposit)\b',
-                                                                  0.85, "Job fee scam",         "Legitimate jobs don't ask you to pay a registration fee upfront."),
+             0.85, "Job fee scam",
+             "Legitimate jobs don't ask you to pay a registration fee upfront."),
             (r'\b(shortlisted|selected).{0,30}(job|position|role|data\s*entry).{0,30}(register|fee|limited)\b',
-                                                                  0.85, "Fake job shortlisting", "Unsolicited job shortlisting with urgency or a fee is a recruitment scam pattern."),
-
+             0.85, "Fake job shortlisting",
+             "Unsolicited job shortlisting with urgency or a fee is a recruitment scam."),
             # Investment fraud
             (r'\b(invest.{0,20}(return|profit|earning)).{0,30}(guaranteed|assured|fixed)\b',
-                                                                  0.88, "Investment scam",      "Guaranteed returns on investments don't exist — this is a hallmark of financial fraud."),
+             0.88, "Investment scam",
+             "Guaranteed returns don't exist -- hallmark of financial fraud."),
             (r'\bdouble.{0,15}(money|investment|amount|profit)\b',
-                                                                  0.90, "Investment doubling scam", "No legitimate scheme doubles your money. This is a Ponzi/pyramid scam pattern."),
+             0.90, "Investment doubling scam",
+             "No legitimate scheme doubles your money. This is a Ponzi/pyramid scam pattern."),
             (r'\b(crypto|trading|forex).{0,30}(group|signal|profit|return|earn).{0,30}(percent|%|lakh|crore)\b',
-                                                                  0.88, "Crypto trading scam",  "Fake trading groups with guaranteed returns are the 'pig butchering' scam pattern."),
-
+             0.88, "Crypto trading scam",
+             "Fake trading groups with guaranteed returns -- the 'pig butchering' scam pattern."),
             # Phishing links
             (r'\bclick.{0,20}(link|here|below).{0,20}(verify|confirm|update|claim|secure)\b',
-                                                                  0.80, "Phishing link",        "Being directed to click a link to verify or claim something is a phishing setup."),
+             0.80, "Phishing link",
+             "Being directed to click a link to verify or claim something is a phishing setup."),
             (r'\b(update|confirm).{0,20}(personal|bank|card|account)\s*(detail|info|data)\b',
-                                                                  0.85, "Data harvesting",      "Requests to 'update' personal or financial details via a link are data theft attempts."),
-
+             0.85, "Data harvesting",
+             "Requests to 'update' personal or financial details via a link are data theft attempts."),
             # Customs/Courier fraud
             (r'\b(customs|clearance).{0,30}(charge|fee|pay|pending|release)\b',
-                                                                  0.85, "Customs fee scam",     "Customs 'clearance fees' requested via SMS are fake — official notices come through courier channels."),
+             0.85, "Customs fee scam",
+             "Customs clearance fees via SMS are fake -- official notices come through couriers."),
             (r'\b(parcel|package|shipment|courier).{0,30}(stuck|hold|pending|failed).{0,30}(pay|fee|charge|verify)\b',
-                                                                  0.80, "Courier fraud",        "Delivery failure messages asking you to pay or verify details are typically fraudulent."),
-
+             0.80, "Courier fraud",
+             "Delivery failure messages asking you to pay or verify details are typically fraudulent."),
             # Tech support
             (r'\b(microsoft|apple|google).{0,30}(security|malicious|virus|malware|traffic).{0,30}(install|call|contact)\b',
-                                                                  0.88, "Tech support scam",    "Microsoft/Apple/Google don't contact you about malware on your device via SMS."),
+             0.88, "Tech support scam",
+             "Microsoft/Apple/Google don't contact you about malware via SMS."),
         ]
 
-        logger.info("Model ready.")
+        logger.info("All models ready.")
+
+    # ── Text model training ───────────────────────────────────────────────────
 
-    def train_model(self):
-        """Train on SMS Spam + new scam dataset combined."""
+    def train_text_model(self):
         texts, labels = [], []
 
-        # Load SMS spam dataset if available
         try:
             sms_df = pd.read_csv("spam.csv", encoding='latin-1')[['v1', 'v2']]
             sms_df.columns = ['label_raw', 'text']
@@ -140,238 +235,357 @@ class ScamDetectionService:
             labels += list(sms_df['label'])
             logger.info(f"Loaded {len(sms_df)} SMS spam samples.")
         except FileNotFoundError:
-            logger.warning("spam.csv not found, training on new dataset only.")
+            logger.warning("spam.csv not found.")
 
-        # Load new scam dataset
         try:
-            new_df = pd.read_csv("scam_detection_training_dataset.csv", encoding='latin-1')
+            new_df = pd.read_csv("scam_messages_complete_500.csv", encoding='latin-1')
             new_df.columns = [c.lower().strip() for c in new_df.columns]
             label_map = {
-                'SCAM': 1,
+                'SCAM':                      1,
                 'LOOKS_GOOD_BUT_SUSPICIOUS': 1,
-                'SUSPICIOUS': 1,
-                'FISHY_BUT_LEGITIMATE': 0,
-                'LEGITIMATE': 0,
+                'SUSPICIOUS':                1,
+                'FISHY_BUT_LEGITIMATE':      0,
+                'LEGITIMATE':                0,
             }
             new_df['label_int'] = new_df['label'].map(label_map)
             new_df = new_df.dropna(subset=['label_int'])
-            # Upweight new dataset 5x (it's smaller but high quality)
-            texts += list(new_df['message_text']) * 5
+            texts  += list(new_df['message_text']) * 5
             labels += list(new_df['label_int'].astype(int)) * 5
-            logger.info(f"Loaded {len(new_df)} new scam samples.")
+            logger.info(f"Loaded {len(new_df)} India-specific scam samples (5x upweighted).")
         except FileNotFoundError:
-            logger.warning("scam_detection_training_dataset.csv not found.")
+            logger.warning("scam_messages_complete_500.csv not found.")
 
         if not texts:
-            raise RuntimeError("No training data found. Add spam.csv or scam_detection_training_dataset.csv.")
+            raise RuntimeError("No training data found.")
 
-        self.model = Pipeline([
+        self.text_model = Pipeline([
             ('tfidf', TfidfVectorizer(
                 max_features=5000,
                 ngram_range=(1, 2),
                 stop_words='english',
                 min_df=1,
-                sublinear_tf=True
+                sublinear_tf=True,
             )),
             ('clf', LogisticRegression(
                 max_iter=1000,
                 C=1.0,
-                class_weight='balanced'
+                class_weight='balanced',
             ))
         ])
 
-        logger.info(f"Training on {len(texts)} total samples...")
-        self.model.fit(texts, labels)
+        logger.info(f"Training text model on {len(texts)} samples...")
+        self.text_model.fit(texts, labels)
 
         with open("spam_model.pkl", 'wb') as f:
-            pickle.dump(self.model, f)
+            pickle.dump(self.text_model, f)
+        logger.info("Text model saved -> spam_model.pkl")
 
-        logger.info("Model trained and saved.")
+    # ── URL ensemble training ─────────────────────────────────────────────────
 
-    def detect_language(self, text: str) -> str:
-        if re.search(r'[\u0900-\u097F]', text):
-            return 'hi'
-        elif re.search(r'[\u0980-\u09FF]', text):
-            return 'or'
-        return 'en'
+    def train_url_ensemble(self):
+        """
+        Three-model soft-voting ensemble on 250-row labeled URL dataset.
 
-    def get_language_confidence(self, text: str) -> float:
-        wc = len(text.split())
-        return 0.6 if wc < 5 else (0.8 if wc < 15 else 0.9)
+        Model A: char n-gram TF-IDF on URL string -> Logistic Regression
+                 Learns character-level patterns (e.g. 'hdfc-', '-kyc', '.xyz').
 
-    def _check_social_engineering(self, text: str) -> Tuple[float, List[str], List[str]]:
-        """
-        Returns (max_risk_score, list_of_short_reasons, list_of_detailed_reasons).
+        Model B: 21 engineered numerical features -> Random Forest
+                 Learns structural signals: hyphen count, TLD type, subdomain depth.
+
+        Model C: word TF-IDF on (url + red_flags + domain_pattern) -> XGBoost
+                 Learns combined text+signal keyword interactions.
+
+        Final score: average of three probability vectors.
+        Override: if any single model has >= 0.85 confidence on MALICIOUS, use it.
         """
+        try:
+            df = pd.read_csv("scam_urls_training_250.csv", encoding='latin-1')
+        except FileNotFoundError:
+            logger.warning("scam_urls_training_250.csv not found. URL ensemble disabled.")
+            self.url_ensemble = None
+            return
+
+        label_map = {'LEGITIMATE': 0, 'SUSPICIOUS': 1, 'MALICIOUS': 2}
+        df['label_int'] = df['label'].map(label_map)
+        df = df.dropna(subset=['label_int'])
+        df['label_int'] = df['label_int'].astype(int)
+
+        urls   = df['url'].tolist()
+        labels = df['label_int'].tolist()
+
+        # Model A: char n-gram TF-IDF + LR
+        model_a = Pipeline([
+            ('tfidf', TfidfVectorizer(
+                analyzer='char_wb',
+                ngram_range=(3, 5),
+                max_features=3000,
+                sublinear_tf=True,
+            )),
+            ('clf', LogisticRegression(
+                max_iter=1000,
+                C=1.0,
+                class_weight='balanced',
+            ))
+        ])
+
+        # Model B: engineered features + RF
+        feat_matrix = np.array([
+            list(extract_url_features(u).values()) for u in urls
+        ], dtype=float)
+
+        model_b = RandomForestClassifier(
+            n_estimators=200,
+            max_depth=8,
+            class_weight='balanced',
+            random_state=42,
+        )
+
+        # Model C: combined text + XGBoost
+        combined_text = [
+            f"{row['url']} {row.get('red_flags', '')} {row.get('domain_pattern', '')}"
+            for _, row in df.iterrows()
+        ]
+
+        tfidf_c = TfidfVectorizer(
+            analyzer='word',
+            ngram_range=(1, 2),
+            max_features=2000,
+            sublinear_tf=True,
+        )
+        feat_c = tfidf_c.fit_transform(combined_text)
+
+        model_c = XGBClassifier(
+            n_estimators=150,
+            max_depth=4,
+            learning_rate=0.1,
+            eval_metric='mlogloss',
+            random_state=42,
+            verbosity=0,
+        )
+
+        logger.info("Training Model A (char TF-IDF + LR)...")
+        model_a.fit(urls, labels)
+        cv_a = cross_val_score(model_a, urls, labels, cv=3, scoring='balanced_accuracy').mean()
+        logger.info(f"  Model A 3-fold balanced accuracy: {cv_a:.3f}")
+
+        logger.info("Training Model B (engineered features + RF)...")
+        model_b.fit(feat_matrix, labels)
+        cv_b = cross_val_score(model_b, feat_matrix, labels, cv=3, scoring='balanced_accuracy').mean()
+        logger.info(f"  Model B 3-fold balanced accuracy: {cv_b:.3f}")
+
+        logger.info("Training Model C (combined text + XGBoost)...")
+        model_c.fit(feat_c, labels)
+        cv_c = cross_val_score(model_c, feat_c, labels, cv=3, scoring='balanced_accuracy').mean()
+        logger.info(f"  Model C 3-fold balanced accuracy: {cv_c:.3f}")
+
+        self.url_ensemble = {
+            'model_a':              model_a,
+            'model_b':              model_b,
+            'model_b_feat_names':   list(extract_url_features(urls[0]).keys()),
+            'model_c_tfidf':        tfidf_c,
+            'model_c':              model_c,
+            'label_map_inv':        {0: 'LEGITIMATE', 1: 'SUSPICIOUS', 2: 'MALICIOUS'},
+            'cv_scores':            {'model_a': cv_a, 'model_b': cv_b, 'model_c': cv_c},
+        }
+
+        with open("url_ensemble.pkl", 'wb') as f:
+            pickle.dump(self.url_ensemble, f)
+        logger.info(
+            f"URL ensemble saved -> url_ensemble.pkl  "
+            f"(A={cv_a:.3f}, B={cv_b:.3f}, C={cv_c:.3f})"
+        )
+
+    # ── Helpers ───────────────────────────────────────────────────────────────
+
+    def _check_social_engineering(self, text: str) -> Tuple[float, List[str], List[str]]:
         text_lower = text.lower()
         short_reasons, detailed_reasons = [], []
         max_score = 0.0
-
         for pattern, score, short, detail in self.social_engineering_patterns:
             if re.search(pattern, text_lower):
                 if short not in short_reasons:
                     short_reasons.append(short)
                     detailed_reasons.append(detail)
                 max_score = max(max_score, score)
-
         return max_score, short_reasons, detailed_reasons
 
+    def detect_language(self, text: str) -> str:
+        if re.search(r'[\u0900-\u097F]', text):
+            return 'hi'
+        elif re.search(r'[\u0980-\u09FF]', text):
+            return 'or'
+        return 'en'
+
+    def _ensemble_url_predict(self, url: str, red_flags_hint: str = "") -> Tuple[float, float, float]:
+        """Returns (p_legitimate, p_suspicious, p_malicious). Falls back to safe if ensemble missing."""
+        if not self.url_ensemble:
+            return 1.0, 0.0, 0.0
+
+        e        = self.url_ensemble
+        combined = f"{url} {red_flags_hint}"
+
+        pa = e['model_a'].predict_proba([url])[0]
+
+        feats = np.array([list(extract_url_features(url).values())], dtype=float)
+        pb    = e['model_b'].predict_proba(feats)[0]
+
+        feat_c = e['model_c_tfidf'].transform([combined])
+        pc     = e['model_c'].predict_proba(feat_c)[0]
+
+        avg = (pa + pb + pc) / 3.0
+
+        # Single-model override if very confident on MALICIOUS
+        for probs in [pa, pb, pc]:
+            if probs[2] >= 0.85:
+                avg = probs
+                break
+
+        return float(avg[0]), float(avg[1]), float(avg[2])
+
+    # ── Text analysis ─────────────────────────────────────────────────────────
+
     def analyze_text_scam(self, text: str, language: str = None) -> Dict:
         if not text or not text.strip():
             return {
-                "risk_level": "Safe",
-                "confidence": 0.0,
-                "reasoning": "Empty text provided.",
-                "user_message": "Nothing to analyze.",
-                "detected_language": "unknown"
+                "risk_level": "Safe", "confidence": 0.0,
+                "reasoning": "Empty text.", "user_message": "Nothing to analyze.",
+                "detected_language": "unknown",
             }
 
         detected_language = language or self.detect_language(text)
 
         try:
-            proba = self.model.predict_proba([text])[0]
-            spam_prob = proba[1]
-
+            proba      = self.text_model.predict_proba([text])[0]
+            spam_prob  = proba[1]
             se_score, se_short, se_detailed = self._check_social_engineering(text)
+            effective  = max(spam_prob, se_score)
 
-            # Final score = max of ML and rule-based (conservative)
-            effective_score = max(spam_prob, se_score)
-
-            if effective_score >= 0.55:
-                risk_level = "Scam"
-                confidence = effective_score
-            elif effective_score >= 0.35:
-                risk_level = "Suspicious"
-                confidence = effective_score
+            if effective >= 0.55:
+                risk_level, confidence = "Scam",       effective
+            elif effective >= 0.35:
+                risk_level, confidence = "Suspicious",  effective
             else:
-                risk_level = "Safe"
-                confidence = 1 - effective_score
+                risk_level, confidence = "Safe",        1 - effective
 
-            # Build reasoning string (for API consumers)
-            reasoning_parts = [f"Spam probability: {round(spam_prob * 100, 1)}%"]
+            reasoning = f"Spam probability: {round(spam_prob * 100, 1)}%"
             if se_short:
-                reasoning_parts.append(f"Flags: {', '.join(se_short[:3])}")
-            reasoning = " | ".join(reasoning_parts)
+                reasoning += f" | Flags: {', '.join(se_short[:3])}"
 
-            # Build human-readable message for UI
             if risk_level == "Scam":
                 if se_detailed:
-                    detail = se_detailed[0]
                     extra = f" Also flagged: {', '.join(se_short[1:3])}." if len(se_short) > 1 else ""
-                    user_message = f"{detail}{extra} Do not share any details or click any links."
+                    user_message = f"{se_detailed[0]}{extra} Do not share any details or click any links."
                 else:
-                    user_message = f"Model confidence {round(spam_prob*100)}% spam. Multiple scam signals detected. Do not share any details or click any links."
-
+                    user_message = f"Model confidence {round(spam_prob*100)}%. Multiple scam signals detected. Do not share details or click links."
             elif risk_level == "Suspicious":
                 if se_detailed:
-                    detail = se_detailed[0]
-                    flags = ", ".join(se_short[:3])
-                    user_message = f"Flagged for: {flags}. {detail} Verify independently before responding."
+                    user_message = f"Flagged for: {', '.join(se_short[:3])}. {se_detailed[0]} Verify independently before responding."
                 else:
-                    user_message = f"Some characteristics of this message match spam patterns (score: {round(spam_prob*100)}%). Worth a second look before you respond."
-
-            else:  # Safe
-                user_message = "No scam signals detected. That said, always be cautious — if something feels off, trust your gut and verify through official channels."
+                    user_message = f"Some characteristics match spam patterns (score: {round(spam_prob*100)}%). Worth a second look."
+            else:
+                user_message = "No scam signals detected. Still be cautious -- if something feels off, verify through official channels."
 
             return {
-                "risk_level": risk_level,
-                "confidence": round(confidence, 4),
-                "reasoning": reasoning,
-                "user_message": user_message,
-                "detected_language": detected_language
+                "risk_level":        risk_level,
+                "confidence":        round(confidence, 4),
+                "reasoning":         reasoning,
+                "user_message":      user_message,
+                "detected_language": detected_language,
             }
-
         except Exception as e:
-            logger.error(f"Classification failed: {e}")
+            logger.error(f"Text classification failed: {e}")
             return {
-                "risk_level": "Suspicious",
-                "confidence": 0.5,
-                "reasoning": f"Model error: {str(e)}",
+                "risk_level": "Suspicious", "confidence": 0.5,
+                "reasoning": f"Model error: {e}",
                 "user_message": "Could not analyze this message. Treat with caution.",
-                "detected_language": detected_language
+                "detected_language": detected_language,
             }
 
+    # ── URL analysis (ensemble + rules blended) ───────────────────────────────
+
     def analyze_url_scam(self, url: str, context: str = "") -> Dict:
         if not url:
             return {
-                "risk_level": "Safe",
-                "confidence": 0.0,
-                "reasoning": "No URL provided.",
-                "user_message": "Nothing to analyze.",
-                "domain": "",
-                "url_status": "invalid"
+                "risk_level": "Safe", "confidence": 0.0,
+                "reasoning": "No URL provided.", "user_message": "Nothing to analyze.",
+                "domain": "", "url_status": "invalid",
             }
 
         try:
             if not url.startswith(('http://', 'https://')):
                 url = 'http://' + url
 
-            parsed_url = urlparse(url)
-            domain = parsed_url.netloc.lower()
+            domain = urlparse(url).netloc.lower()
 
-            rule_risk = 0.0
-            rule_flags = []  # (short_label, explanation)
+            # Rule layer
+            rule_risk  = 0.0
+            rule_flags = []
 
-            # Protocol check
-            is_http = url.startswith('http://') and not url.startswith('https://')
-            if is_http:
+            if url.startswith('http://') and not url.startswith('https://'):
                 rule_risk += 0.30
-                rule_flags.append(("HTTP not HTTPS", "Connection is unencrypted (HTTP). Any data you enter can be intercepted."))
+                rule_flags.append(("HTTP not HTTPS",
+                                   "Connection is unencrypted. Any data you enter can be intercepted."))
 
-            # Brand spoofing
             scam_domain_patterns = [
                 "faceb00k", "paypa1", "amaz0n", "micros0ft", "g00gle",
                 "appleid", "login-secure", "claim-your", "verify-account",
                 "lottery", "techsupport", "quickloan", "account-update",
                 "hdfc-", "sbi-", "icici-", "paytm-", "netflix-payment",
-                "bluedart-track", "india-post"
+                "bluedart-track", "india-post",
             ]
-            matched_brands = [p for p in scam_domain_patterns if p in domain]
-            if matched_brands:
+            matched = [p for p in scam_domain_patterns if p in domain]
+            if matched:
                 rule_risk += 0.85
-                rule_flags.append(("Brand spoofing", f"Domain appears to impersonate a trusted brand ({matched_brands[0]}). The real service would use its official domain."))
+                rule_flags.append(("Brand spoofing",
+                                   f"Domain impersonates a trusted brand ({matched[0]}). Use the official domain."))
 
-            # URL shorteners
-            matched_shortener = next((s for s in self.suspicious_shorteners if s in domain), None)
-            if matched_shortener:
+            shortener = next((s for s in self.suspicious_shorteners if s in domain), None)
+            if shortener:
                 rule_risk += 0.55
-                rule_flags.append(("URL shortener", f"Uses {matched_shortener} — hides the real destination. You have no way to know where this link actually goes."))
+                rule_flags.append(("URL shortener",
+                                   f"Uses {shortener} -- hides the real destination."))
 
-            # Raw IP address
             if re.search(r'\d+\.\d+\.\d+\.\d+', domain):
                 rule_risk += 0.90
-                rule_flags.append(("Raw IP address", "Uses an IP address instead of a domain name. Legitimate services never do this."))
+                rule_flags.append(("Raw IP address",
+                                   "Legitimate services never use raw IP addresses."))
 
-            # Suspicious TLDs
-            suspicious_tlds = ['.tk', '.ml', '.ga', '.cf', '.gq', '.xyz', '.top', '.pw', '.click', '.info', '.site']
+            suspicious_tlds = ['.tk', '.ml', '.ga', '.cf', '.gq', '.xyz',
+                               '.top', '.pw', '.click', '.info', '.site']
             matched_tld = next((t for t in suspicious_tlds if domain.endswith(t)), None)
             if matched_tld:
                 rule_risk += 0.65
-                rule_flags.append(("Suspicious TLD", f"'{matched_tld}' is a free or low-cost domain commonly used in phishing campaigns."))
+                rule_flags.append(("Suspicious TLD",
+                                   f"'{matched_tld}' is commonly used in phishing campaigns."))
 
-            # Long URL
             if len(url) > 100:
                 rule_risk += 0.25
-                rule_flags.append(("Abnormally long URL", "Very long URLs with many parameters are a common obfuscation tactic."))
+                rule_flags.append(("Abnormally long URL",
+                                   "Very long URLs with many parameters are a common obfuscation tactic."))
 
-            # Security keywords in URL
-            security_words = ['login', 'verify', 'update', 'confirm', 'secure', 'account', 'signin', 'banking']
+            security_words = ['login', 'verify', 'update', 'confirm',
+                              'secure', 'account', 'signin', 'banking']
             matched_kw = [w for w in security_words if w in url.lower()]
             if matched_kw:
                 rule_risk += 0.35
-                rule_flags.append(("Security keywords in URL", f"Contains '{matched_kw[0]}' — phishing pages often use these words to appear legitimate."))
+                rule_flags.append(("Security keywords in URL",
+                                   f"Contains '{matched_kw[0]}' -- phishing pages use these to appear legitimate."))
+
+            # Ensemble prediction
+            p_legit, p_susp, p_mal = self._ensemble_url_predict(url)
+            ensemble_risk = p_mal * 1.0 + p_susp * 0.5
+
+            # 50/50 blend
+            final_score = min(0.50 * min(rule_risk, 1.0) + 0.50 * ensemble_risk, 1.0)
 
-            # Analyze context if provided
-            ml_risk = 0.0
+            # Context boost from message analysis
             context_result = None
             if context:
                 context_result = self.analyze_text_scam(context)
                 if context_result['risk_level'] == 'Scam':
-                    ml_risk = context_result['confidence']
+                    final_score = min(final_score + 0.15, 1.0)
                 elif context_result['risk_level'] == 'Suspicious':
-                    ml_risk = context_result['confidence'] * 0.6
-
-            final_score = min((rule_risk * 0.65) + (ml_risk * 0.35), 1.0)
+                    final_score = min(final_score + 0.07, 1.0)
 
             if final_score >= 0.55:
                 risk_level = "Scam"
@@ -380,55 +594,51 @@ class ScamDetectionService:
             else:
                 risk_level = "Safe"
 
-            # Build API reasoning
             flag_labels = [f[0] for f in rule_flags]
-            reasoning = "; ".join(flag_labels) if flag_labels else "No suspicious patterns"
+            reasoning   = "; ".join(flag_labels) if flag_labels else "No rule flags"
+            reasoning  += (f" | Ensemble: {round(p_mal*100)}% malicious, "
+                           f"{round(p_susp*100)}% suspicious")
             if context_result:
                 reasoning += f" | Message context: {context_result['risk_level']}"
 
-            # Build UI message
             if risk_level == "Scam":
-                if rule_flags:
-                    primary = rule_flags[0][1]
-                    extras = [f[0] for f in rule_flags[1:3]]
-                    extra_str = f" Also: {', '.join(extras)}." if extras else ""
-                    user_message = f"{primary}{extra_str} Do not open this link."
-                else:
-                    user_message = "This URL has multiple high-risk indicators. Do not open it."
-
+                primary = rule_flags[0][1] if rule_flags else "High-risk URL detected by ensemble classifier."
+                extras  = [f[0] for f in rule_flags[1:3]]
+                extra_s = f" Also: {', '.join(extras)}." if extras else ""
+                user_message = f"{primary}{extra_s} Do not open this link."
             elif risk_level == "Suspicious":
-                if rule_flags:
-                    primary = rule_flags[0][1]
-                    user_message = f"{primary} Verify this URL is from an official source before clicking."
-                else:
-                    user_message = "This URL has some unusual characteristics. Verify before clicking."
-
+                primary = rule_flags[0][1] if rule_flags else "URL has unusual structural characteristics."
+                user_message = f"{primary} Verify this is from an official source before clicking."
             else:
                 user_message = "No major red flags detected. Still, only click links from sources you initiated contact with."
 
             return {
-                "risk_level": risk_level,
-                "confidence": round(final_score, 4),
-                "reasoning": reasoning,
-                "user_message": user_message,
-                "domain": domain,
-                "url_status": "analyzed"
+                "risk_level":      risk_level,
+                "confidence":      round(final_score, 4),
+                "reasoning":       reasoning,
+                "user_message":    user_message,
+                "domain":          domain,
+                "url_status":      "analyzed",
+                "ensemble_scores": {
+                    "p_legitimate": round(p_legit, 3),
+                    "p_suspicious": round(p_susp, 3),
+                    "p_malicious":  round(p_mal, 3),
+                },
             }
 
         except Exception as e:
+            logger.error(f"URL analysis error: {e}")
             return {
-                "risk_level": "Suspicious",
-                "confidence": 0.5,
-                "reasoning": f"URL analysis error: {str(e)}",
+                "risk_level": "Suspicious", "confidence": 0.5,
+                "reasoning": f"URL analysis error: {e}",
                 "user_message": "Could not fully analyze this URL. Treat with caution.",
-                "domain": "unknown",
-                "url_status": "error"
+                "domain": "unknown", "url_status": "error",
             }
 
     def generate_user_response(self, risk_level: str) -> str:
         responses = {
-            "Safe": "✅ This message appears safe.",
-            "Suspicious": "⚠️ Be cautious — this message has suspicious elements.",
-            "Scam": "🚨 WARNING: This appears to be a scam! Do not click links or share personal info."
+            "Safe":       "This message appears safe.",
+            "Suspicious": "Be cautious -- this message has suspicious elements.",
+            "Scam":       "WARNING: This appears to be a scam! Do not click links or share personal info.",
         }
         return responses.get(risk_level, "Unable to analyze.")