ml_utils.py

# ml_utils.py, adding ensemble for URL scam detection

import re
import pickle
import logging
import numpy as np
from typing import Dict, List, Tuple
from urllib.parse import urlparse
from pathlib import Path

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from xgboost import XGBClassifier

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


# ── URL feature engineering ───────────────────────────────────────────────────

def extract_url_features(url: str) -> Dict:
    """Extract structured numerical features from a URL string."""
    try:
        raw = url
        if not url.startswith(('http://', 'https://')):
            url = 'http://' + url
        p = urlparse(url)
        domain = p.netloc.lower()
        path = p.path.lower()
        full = url.lower()

        SECURITY_KW = ['login', 'verify', 'update', 'confirm', 'secure',
                       'account', 'kyc', 'banking', 'signin', 'validation']
        BRAND_KW    = ['hdfc', 'sbi', 'icici', 'paytm', 'amazon', 'google',
                       'microsoft', 'paypal', 'netflix', 'airtel', 'jio',
                       'phonepe', 'razorpay', 'flipkart', 'swiggy', 'zomato',
                       'axis', 'kotak', 'uidai', 'npci', 'irctc']
        SUSP_TLDS   = ['.tk', '.ml', '.ga', '.cf', '.gq', '.xyz', '.top',
                       '.pw', '.click', '.site', '.co', '.in']
        FREE_TLDS   = ['.tk', '.ml', '.ga', '.cf', '.gq']

        return {
            'length':                int(len(raw)),
            'dot_count':             int(raw.count('.')),
            'hyphen_count':          int(raw.count('-')),
            'slash_count':           int(raw.count('/')),
            'digit_count':           int(sum(c.isdigit() for c in domain)),
            'is_https':              int(url.startswith('https://')),
            'is_ip':                 int(bool(re.search(r'\d+\.\d+\.\d+\.\d+', domain))),
            'subdomain_depth':       int(max(len(domain.split('.')) - 2, 0)),
            'has_security_kw':       int(any(w in full for w in SECURITY_KW)),
            'has_brand_kw':          int(any(w in full for w in BRAND_KW)),
            'suspicious_tld':        int(any(domain.endswith(t) for t in SUSP_TLDS)),
            'has_free_tld':          int(any(domain.endswith(t) for t in FREE_TLDS)),
            'path_length':           int(len(p.path)),
            'has_numbers_in_domain': int(bool(re.search(r'\d', domain.split('.')[0]))),
            'hyphen_in_domain':      int('-' in domain),
            'multi_hyphens':         int(domain.count('-') >= 2),
            'at_sign':               int('@' in url),
            'double_slash_redirect': int('//' in p.path),
            'query_length':          int(len(p.query)),
            'brand_plus_hyphen':     int(any(w in domain and '-' in domain for w in BRAND_KW)),
            'security_kw_in_path':   int(any(w in path for w in SECURITY_KW)),
        }
    except Exception:
        return {k: 0 for k in [
            'length', 'dot_count', 'hyphen_count', 'slash_count', 'digit_count',
            'is_https', 'is_ip', 'subdomain_depth', 'has_security_kw', 'has_brand_kw',
            'suspicious_tld', 'has_free_tld', 'path_length', 'has_numbers_in_domain',
            'hyphen_in_domain', 'multi_hyphens', 'at_sign', 'double_slash_redirect',
            'query_length', 'brand_plus_hyphen', 'security_kw_in_path',
        ]}


class ScamDetectionService:
    def __init__(self):
        logger.info("Loading models...")

        text_path = Path("spam_model.pkl")
        url_path  = Path("url_ensemble.pkl")

        if text_path.exists():
            with open(text_path, 'rb') as f:
                self.text_model = pickle.load(f)
            logger.info("Text model loaded.")
        else:
            logger.info("Training text model...")
            self.train_text_model()

        if url_path.exists():
            with open(url_path, 'rb') as f:
                self.url_ensemble = pickle.load(f)
            logger.info("URL ensemble loaded.")
        else:
            logger.info("Training URL ensemble...")
            self.train_url_ensemble()

        self.suspicious_shorteners = [
            'bit.ly', 'tinyurl.com', 'short.link', 'tiny.cc', 'ow.ly',
            'goo.gl', 't.co', 'rb.gy', 'is.gd', 'v.gd', 'cutt.ly'
        ]

        # (regex, risk_score, short_label, detailed_explanation)
        self.social_engineering_patterns: List[Tuple[str, float, str, str]] = [
            # OTP / credential harvesting
            (r'\bshare.{0,20}otp\b',                              0.90, "OTP request",
             "Asks you to share an OTP -- no legitimate org ever does this."),
            (r'\bconfirm.{0,20}(otp|pin|password)\b',            0.85, "Credential request",
             "Requests you confirm an OTP, PIN, or password -- classic phishing."),
            (r'\b(last\s*4|last\s*four).{0,20}(digit|card)\b',   0.85, "Card digit request",
             "Asks for last 4 digits of your card -- used to build to full card theft."),
            (r'\b(card\s*number|cvv|expiry)\b',                   0.85, "Card detail request",
             "Requests card number, CVV, or expiry -- your bank will never ask over SMS."),
            (r'\botp\b',                                          0.45, "OTP mention",
             "Mentions OTP -- context suggests a transaction or verification prompt."),
            # Bank / govt impersonation
            (r'\b(bank|rbi|sbi|hdfc|icici|axis|kotak|pnb|boi).{0,30}(suspend|block|close|deactivat|restrict)\b',
             0.88, "Bank account threat",
             "Claims your bank account is being suspended -- banks use official mail, not SMS links."),
            (r'\b(fraud\s*prevention|fraud\s*team|fraud\s*department|fraud\s*monitoring)\b',
             0.75, "Fraud team impersonation",
             "Impersonates a bank fraud team to create panic and urgency."),
            (r'\b(kyc|know\s*your\s*customer).{0,20}(update|expire|pending|complet|verif)\b',
             0.88, "KYC scam",
             "KYC update requests via SMS are almost always fraudulent."),
            (r'\baadhaar.{0,30}(link|update|verify|expire|deactivat|biometric)\b',
             0.85, "Aadhaar scam",
             "UIDAI does not send Aadhaar deactivation or verification requests via SMS."),
            (r'\b(pan\s*card|pan).{0,30}(flag|block|verify|link|update)\b',
             0.82, "PAN card scam",
             "PAN verification is done only through the income tax portal -- not SMS links."),
            (r'\b(rbi|reserve\s*bank).{0,30}(notice|compliance|regulat|review|audit)\b',
             0.85, "RBI impersonation",
             "The RBI does not contact individuals directly via SMS for compliance or audits."),
            (r'\b(income\s*tax|it\s*department|tds).{0,30}(refund|notice|verif|confirm|scrutin)\b',
             0.85, "Tax dept scam",
             "Income tax refunds and notices come through e-Filing portal, not SMS."),
            (r'\b(gst|epfo|uan|pf\s*deposit).{0,30}(verif|update|link|suspend|block)\b',
             0.82, "Govt portal impersonation",
             "Legitimate EPFO/GST communications don't ask for verification via SMS links."),
            # Telecom
            (r'\b(sim|mobile).{0,30}(deactivat|block|suspend|port).{0,20}(kyc|verif|update)\b',
             0.85, "SIM KYC scam",
             "TRAI and telecom operators don't deactivate SIMs via SMS verification links."),
            (r'\b(airtel|jio|vi|vodafone|bsnl).{0,30}(block|suspend|deactivat|kyc|port)\b',
             0.82, "Telecom impersonation",
             "Telecom providers handle SIM issues at stores or official apps -- not SMS links."),
            # Digital arrest / legal threat
            (r'\b(cbi|cybercrime|enforcement\s*directorate|ed|police|court).{0,40}(case|notice|filed|investigation|arrest|prosecution)\b',
             0.90, "Law enforcement impersonation",
             "CBI/ED/Police do not initiate legal proceedings via SMS. This is a 'digital arrest' scam."),
            (r'\b(section\s*420|fema|pmla|ipc|money\s*laundering).{0,40}(invest|notice|case|compli)\b',
             0.88, "Legal threat scam",
             "Citing specific legal sections over SMS to create fear is a known fraud tactic."),
            (r'\b(legal\s*notice|warrant|fir)\b',
             0.80, "Legal threat",
             "Legitimate legal notices arrive through official postal or court channels, not SMS."),
            # Account suspension / urgency
            (r'\b(account|service|upi|wallet).{0,20}(suspend|block|terminat|deactivat|restrict)\b',
             0.75, "Account suspension threat",
             "Suspension threats via SMS are pressure tactics to make you act without thinking."),
            (r'\b(immediate|immediately|urgent|urgently).{0,30}(action|call|contact|verify|confirm)\b',
             0.70, "Urgency pressure",
             "Manufactured urgency is the #1 social engineering tactic -- bypasses rational thinking."),
            (r'\bwithin\s*(1|2|24|30|48|72)\s*hours?\b',
             0.65, "Time pressure",
             "Artificial deadlines pressure you into acting before you can verify."),
            # Fake helpline
            (r'\b(call|contact).{0,20}(helpline|support|team|officer|number).{0,30}\d{8,12}\b',
             0.75, "Fake helpline",
             "Scammers publish fake helpline numbers -- always call from the official website."),
            # Prize / lottery
            (r'\b(won|win|winner|winning).{0,30}(prize|lottery|lucky|reward|cash|gift|iphone|samsung)\b',
             0.90, "Lottery/prize scam",
             "You cannot win a lottery you didn't enter. Designed to get your personal details."),
            (r'\bcongratulations.{0,50}(won|selected|chosen|winner|shortlist)\b',
             0.90, "Prize scam",
             "Unsolicited congratulations messages are almost universally fraudulent."),
            (r'\bclaim.{0,20}(prize|reward|cash|gift|money|amount)\b',
             0.85, "Claim prize prompt",
             "Asking you to 'claim' a prize you weren't expecting is a classic advance fee setup."),
            # Job fraud
            (r'\b(work\s*from\s*home|earn.{0,10}per\s*day|daily\s*earning|part\s*time\s*job).{0,40}(register|fee|pay|deposit)\b',
             0.85, "Job fee scam",
             "Legitimate jobs don't ask you to pay a registration fee upfront."),
            (r'\b(shortlisted|selected).{0,30}(job|position|role|data\s*entry).{0,30}(register|fee|limited)\b',
             0.85, "Fake job shortlisting",
             "Unsolicited job shortlisting with urgency or a fee is a recruitment scam."),
            # Investment fraud
            (r'\b(invest.{0,20}(return|profit|earning)).{0,30}(guaranteed|assured|fixed)\b',
             0.88, "Investment scam",
             "Guaranteed returns don't exist -- hallmark of financial fraud."),
            (r'\bdouble.{0,15}(money|investment|amount|profit)\b',
             0.90, "Investment doubling scam",
             "No legitimate scheme doubles your money. This is a Ponzi/pyramid scam pattern."),
            (r'\b(crypto|trading|forex).{0,30}(group|signal|profit|return|earn).{0,30}(percent|%|lakh|crore)\b',
             0.88, "Crypto trading scam",
             "Fake trading groups with guaranteed returns -- the 'pig butchering' scam pattern."),
            # Phishing links
            (r'\bclick.{0,20}(link|here|below).{0,20}(verify|confirm|update|claim|secure)\b',
             0.80, "Phishing link",
             "Being directed to click a link to verify or claim something is a phishing setup."),
            (r'\b(update|confirm).{0,20}(personal|bank|card|account)\s*(detail|info|data)\b',
             0.85, "Data harvesting",
             "Requests to 'update' personal or financial details via a link are data theft attempts."),
            # Customs/Courier fraud
            (r'\b(customs|clearance).{0,30}(charge|fee|pay|pending|release)\b',
             0.85, "Customs fee scam",
             "Customs clearance fees via SMS are fake -- official notices come through couriers."),
            (r'\b(parcel|package|shipment|courier).{0,30}(stuck|hold|pending|failed).{0,30}(pay|fee|charge|verify)\b',
             0.80, "Courier fraud",
             "Delivery failure messages asking you to pay or verify details are typically fraudulent."),
            # Tech support
            (r'\b(microsoft|apple|google).{0,30}(security|malicious|virus|malware|traffic).{0,30}(install|call|contact)\b',
             0.88, "Tech support scam",
             "Microsoft/Apple/Google don't contact you about malware via SMS."),
        ]

        logger.info("All models ready.")

    # ── Text model training ───────────────────────────────────────────────────

    def train_text_model(self):
        texts, labels = [], []

        try:
            sms_df = pd.read_csv("spam.csv", encoding='latin-1')[['v1', 'v2']]
            sms_df.columns = ['label_raw', 'text']
            sms_df['label'] = sms_df['label_raw'].map({'ham': 0, 'spam': 1})
            texts += list(sms_df['text'])
            labels += list(sms_df['label'])
            logger.info(f"Loaded {len(sms_df)} SMS spam samples.")
        except FileNotFoundError:
            logger.warning("spam.csv not found.")

        try:
            new_df = pd.read_csv("scam_messages_complete_500.csv", encoding='latin-1')
            new_df.columns = [c.lower().strip() for c in new_df.columns]
            label_map = {
                'SCAM':                      1,
                'LOOKS_GOOD_BUT_SUSPICIOUS': 1,
                'SUSPICIOUS':                1,
                'FISHY_BUT_LEGITIMATE':      0,
                'LEGITIMATE':                0,
            }
            new_df['label_int'] = new_df['label'].map(label_map)
            new_df = new_df.dropna(subset=['label_int'])
            texts  += list(new_df['message_text']) * 5
            labels += list(new_df['label_int'].astype(int)) * 5
            logger.info(f"Loaded {len(new_df)} India-specific scam samples (5x upweighted).")
        except FileNotFoundError:
            logger.warning("scam_messages_complete_500.csv not found.")

        if not texts:
            raise RuntimeError("No training data found.")

        self.text_model = Pipeline([
            ('tfidf', TfidfVectorizer(
                max_features=5000,
                ngram_range=(1, 2),
                stop_words='english',
                min_df=1,
                sublinear_tf=True,
            )),
            ('clf', LogisticRegression(
                max_iter=1000,
                C=1.0,
                class_weight='balanced',
            ))
        ])

        logger.info(f"Training text model on {len(texts)} samples...")
        self.text_model.fit(texts, labels)

        with open("spam_model.pkl", 'wb') as f:
            pickle.dump(self.text_model, f)
        logger.info("Text model saved -> spam_model.pkl")

    # ── URL ensemble training ─────────────────────────────────────────────────

    def train_url_ensemble(self):
        """
        Three-model soft-voting ensemble on 250-row labeled URL dataset.

        Model A: char n-gram TF-IDF on URL string -> Logistic Regression
                 Learns character-level patterns (e.g. 'hdfc-', '-kyc', '.xyz').

        Model B: 21 engineered numerical features -> Random Forest
                 Learns structural signals: hyphen count, TLD type, subdomain depth.

        Model C: word TF-IDF on (url + red_flags + domain_pattern) -> XGBoost
                 Learns combined text+signal keyword interactions.

        Final score: average of three probability vectors.
        Override: if any single model has >= 0.85 confidence on MALICIOUS, use it.
        """
        try:
            df = pd.read_csv("scam_urls_training_250.csv", encoding='latin-1')
        except FileNotFoundError:
            logger.warning("scam_urls_training_250.csv not found. URL ensemble disabled.")
            self.url_ensemble = None
            return

        label_map = {'LEGITIMATE': 0, 'SUSPICIOUS': 1, 'MALICIOUS': 2}
        df['label_int'] = df['label'].map(label_map)
        df = df.dropna(subset=['label_int'])
        df['label_int'] = df['label_int'].astype(int)

        urls   = df['url'].tolist()
        labels = df['label_int'].tolist()

        # Model A: char n-gram TF-IDF + LR
        model_a = Pipeline([
            ('tfidf', TfidfVectorizer(
                analyzer='char_wb',
                ngram_range=(3, 5),
                max_features=3000,
                sublinear_tf=True,
            )),
            ('clf', LogisticRegression(
                max_iter=1000,
                C=1.0,
                class_weight='balanced',
            ))
        ])

        # Model B: engineered features + RF
        feat_matrix = np.array([
            list(extract_url_features(u).values()) for u in urls
        ], dtype=float)

        model_b = RandomForestClassifier(
            n_estimators=200,
            max_depth=8,
            class_weight='balanced',
            random_state=42,
        )

        # Model C: combined text + XGBoost
        combined_text = [
            f"{row['url']} {row.get('red_flags', '')} {row.get('domain_pattern', '')}"
            for _, row in df.iterrows()
        ]

        tfidf_c = TfidfVectorizer(
            analyzer='word',
            ngram_range=(1, 2),
            max_features=2000,
            sublinear_tf=True,
        )
        feat_c = tfidf_c.fit_transform(combined_text)

        model_c = XGBClassifier(
            n_estimators=150,
            max_depth=4,
            learning_rate=0.1,
            eval_metric='mlogloss',
            random_state=42,
            verbosity=0,
        )

        logger.info("Training Model A (char TF-IDF + LR)...")
        model_a.fit(urls, labels)
        cv_a = cross_val_score(model_a, urls, labels, cv=3, scoring='balanced_accuracy').mean()
        logger.info(f"  Model A 3-fold balanced accuracy: {cv_a:.3f}")

        logger.info("Training Model B (engineered features + RF)...")
        model_b.fit(feat_matrix, labels)
        cv_b = cross_val_score(model_b, feat_matrix, labels, cv=3, scoring='balanced_accuracy').mean()
        logger.info(f"  Model B 3-fold balanced accuracy: {cv_b:.3f}")

        logger.info("Training Model C (combined text + XGBoost)...")
        model_c.fit(feat_c, labels)
        cv_c = cross_val_score(model_c, feat_c, labels, cv=3, scoring='balanced_accuracy').mean()
        logger.info(f"  Model C 3-fold balanced accuracy: {cv_c:.3f}")

        self.url_ensemble = {
            'model_a':              model_a,
            'model_b':              model_b,
            'model_b_feat_names':   list(extract_url_features(urls[0]).keys()),
            'model_c_tfidf':        tfidf_c,
            'model_c':              model_c,
            'label_map_inv':        {0: 'LEGITIMATE', 1: 'SUSPICIOUS', 2: 'MALICIOUS'},
            'cv_scores':            {'model_a': cv_a, 'model_b': cv_b, 'model_c': cv_c},
        }

        with open("url_ensemble.pkl", 'wb') as f:
            pickle.dump(self.url_ensemble, f)
        logger.info(
            f"URL ensemble saved -> url_ensemble.pkl  "
            f"(A={cv_a:.3f}, B={cv_b:.3f}, C={cv_c:.3f})"
        )

    # ── Helpers ───────────────────────────────────────────────────────────────

    def _check_social_engineering(self, text: str) -> Tuple[float, List[str], List[str]]:
        text_lower = text.lower()
        short_reasons, detailed_reasons = [], []
        max_score = 0.0
        for pattern, score, short, detail in self.social_engineering_patterns:
            if re.search(pattern, text_lower):
                if short not in short_reasons:
                    short_reasons.append(short)
                    detailed_reasons.append(detail)
                max_score = max(max_score, score)
        return max_score, short_reasons, detailed_reasons

    def detect_language(self, text: str) -> str:
        if re.search(r'[\u0900-\u097F]', text):
            return 'hi'
        elif re.search(r'[\u0980-\u09FF]', text):
            return 'or'
        return 'en'

    def _ensemble_url_predict(self, url: str, red_flags_hint: str = "") -> Tuple[float, float, float]:
        """Returns (p_legitimate, p_suspicious, p_malicious). Falls back to safe if ensemble missing."""
        if not self.url_ensemble:
            return 1.0, 0.0, 0.0

        e        = self.url_ensemble
        combined = f"{url} {red_flags_hint}"

        pa = e['model_a'].predict_proba([url])[0]

        feats = np.array([list(extract_url_features(url).values())], dtype=float)
        pb    = e['model_b'].predict_proba(feats)[0]

        feat_c = e['model_c_tfidf'].transform([combined])
        pc     = e['model_c'].predict_proba(feat_c)[0]

        avg = (pa + pb + pc) / 3.0

        # Single-model override if very confident on MALICIOUS
        for probs in [pa, pb, pc]:
            if probs[2] >= 0.85:
                avg = probs
                break

        return float(avg[0]), float(avg[1]), float(avg[2])

    # ── Text analysis ─────────────────────────────────────────────────────────

    def analyze_text_scam(self, text: str, language: str = None) -> Dict:
        if not text or not text.strip():
            return {
                "risk_level": "Safe", "confidence": 0.0,
                "reasoning": "Empty text.", "user_message": "Nothing to analyze.",
                "detected_language": "unknown",
            }

        detected_language = language or self.detect_language(text)

        try:
            proba      = self.text_model.predict_proba([text])[0]
            spam_prob  = proba[1]
            se_score, se_short, se_detailed = self._check_social_engineering(text)
            effective  = max(spam_prob, se_score)

            if effective >= 0.55:
                risk_level, confidence = "Scam",       effective
            elif effective >= 0.35:
                risk_level, confidence = "Suspicious",  effective
            else:
                risk_level, confidence = "Safe",        1 - effective

            reasoning = f"Spam probability: {round(spam_prob * 100, 1)}%"
            if se_short:
                reasoning += f" | Flags: {', '.join(se_short[:3])}"

            if risk_level == "Scam":
                if se_detailed:
                    extra = f" Also flagged: {', '.join(se_short[1:3])}." if len(se_short) > 1 else ""
                    user_message = f"{se_detailed[0]}{extra} Do not share any details or click any links."
                else:
                    user_message = f"Model confidence {round(spam_prob*100)}%. Multiple scam signals detected. Do not share details or click links."
            elif risk_level == "Suspicious":
                if se_detailed:
                    user_message = f"Flagged for: {', '.join(se_short[:3])}. {se_detailed[0]} Verify independently before responding."
                else:
                    user_message = f"Some characteristics match spam patterns (score: {round(spam_prob*100)}%). Worth a second look."
            else:
                user_message = "No scam signals detected. Still be cautious -- if something feels off, verify through official channels."

            return {
                "risk_level":        risk_level,
                "confidence":        round(confidence, 4),
                "reasoning":         reasoning,
                "user_message":      user_message,
                "detected_language": detected_language,
            }
        except Exception as e:
            logger.error(f"Text classification failed: {e}")
            return {
                "risk_level": "Suspicious", "confidence": 0.5,
                "reasoning": f"Model error: {e}",
                "user_message": "Could not analyze this message. Treat with caution.",
                "detected_language": detected_language,
            }

    # ── URL analysis (ensemble + rules blended) ───────────────────────────────

    def analyze_url_scam(self, url: str, context: str = "") -> Dict:
        if not url:
            return {
                "risk_level": "Safe", "confidence": 0.0,
                "reasoning": "No URL provided.", "user_message": "Nothing to analyze.",
                "domain": "", "url_status": "invalid",
            }

        try:
            if not url.startswith(('http://', 'https://')):
                url = 'http://' + url

            domain = urlparse(url).netloc.lower()

            # Rule layer
            rule_risk  = 0.0
            rule_flags = []

            if url.startswith('http://') and not url.startswith('https://'):
                rule_risk += 0.30
                rule_flags.append(("HTTP not HTTPS",
                                   "Connection is unencrypted. Any data you enter can be intercepted."))

            scam_domain_patterns = [
                "faceb00k", "paypa1", "amaz0n", "micros0ft", "g00gle",
                "appleid", "login-secure", "claim-your", "verify-account",
                "lottery", "techsupport", "quickloan", "account-update",
                "hdfc-", "sbi-", "icici-", "paytm-", "netflix-payment",
                "bluedart-track", "india-post",
            ]
            matched = [p for p in scam_domain_patterns if p in domain]
            if matched:
                rule_risk += 0.85
                rule_flags.append(("Brand spoofing",
                                   f"Domain impersonates a trusted brand ({matched[0]}). Use the official domain."))

            shortener = next((s for s in self.suspicious_shorteners if s in domain), None)
            if shortener:
                rule_risk += 0.55
                rule_flags.append(("URL shortener",
                                   f"Uses {shortener} -- hides the real destination."))

            if re.search(r'\d+\.\d+\.\d+\.\d+', domain):
                rule_risk += 0.90
                rule_flags.append(("Raw IP address",
                                   "Legitimate services never use raw IP addresses."))

            suspicious_tlds = ['.tk', '.ml', '.ga', '.cf', '.gq', '.xyz',
                               '.top', '.pw', '.click', '.info', '.site']
            matched_tld = next((t for t in suspicious_tlds if domain.endswith(t)), None)
            if matched_tld:
                rule_risk += 0.65
                rule_flags.append(("Suspicious TLD",
                                   f"'{matched_tld}' is commonly used in phishing campaigns."))

            if len(url) > 100:
                rule_risk += 0.25
                rule_flags.append(("Abnormally long URL",
                                   "Very long URLs with many parameters are a common obfuscation tactic."))

            security_words = ['login', 'verify', 'update', 'confirm',
                              'secure', 'account', 'signin', 'banking']
            matched_kw = [w for w in security_words if w in url.lower()]
            if matched_kw:
                rule_risk += 0.35
                rule_flags.append(("Security keywords in URL",
                                   f"Contains '{matched_kw[0]}' -- phishing pages use these to appear legitimate."))

            # Ensemble prediction
            p_legit, p_susp, p_mal = self._ensemble_url_predict(url)
            ensemble_risk = p_mal * 1.0 + p_susp * 0.5

            # 50/50 blend
            final_score = min(0.50 * min(rule_risk, 1.0) + 0.50 * ensemble_risk, 1.0)

            # Context boost from message analysis
            context_result = None
            if context:
                context_result = self.analyze_text_scam(context)
                if context_result['risk_level'] == 'Scam':
                    final_score = min(final_score + 0.15, 1.0)
                elif context_result['risk_level'] == 'Suspicious':
                    final_score = min(final_score + 0.07, 1.0)

            if final_score >= 0.55:
                risk_level = "Scam"
            elif final_score >= 0.25:
                risk_level = "Suspicious"
            else:
                risk_level = "Safe"

            flag_labels = [f[0] for f in rule_flags]
            reasoning   = "; ".join(flag_labels) if flag_labels else "No rule flags"
            reasoning  += (f" | Ensemble: {round(p_mal*100)}% malicious, "
                           f"{round(p_susp*100)}% suspicious")
            if context_result:
                reasoning += f" | Message context: {context_result['risk_level']}"

            if risk_level == "Scam":
                primary = rule_flags[0][1] if rule_flags else "High-risk URL detected by ensemble classifier."
                extras  = [f[0] for f in rule_flags[1:3]]
                extra_s = f" Also: {', '.join(extras)}." if extras else ""
                user_message = f"{primary}{extra_s} Do not open this link."
            elif risk_level == "Suspicious":
                primary = rule_flags[0][1] if rule_flags else "URL has unusual structural characteristics."
                user_message = f"{primary} Verify this is from an official source before clicking."
            else:
                user_message = "No major red flags detected. Still, only click links from sources you initiated contact with."

            return {
                "risk_level":      risk_level,
                "confidence":      round(final_score, 4),
                "reasoning":       reasoning,
                "user_message":    user_message,
                "domain":          domain,
                "url_status":      "analyzed",
                "ensemble_scores": {
                    "p_legitimate": round(p_legit, 3),
                    "p_suspicious": round(p_susp, 3),
                    "p_malicious":  round(p_mal, 3),
                },
            }

        except Exception as e:
            logger.error(f"URL analysis error: {e}")
            return {
                "risk_level": "Suspicious", "confidence": 0.5,
                "reasoning": f"URL analysis error: {e}",
                "user_message": "Could not fully analyze this URL. Treat with caution.",
                "domain": "unknown", "url_status": "error",
            }

    def generate_user_response(self, risk_level: str) -> str:
        responses = {
            "Safe":       "This message appears safe.",
            "Suspicious": "Be cautious -- this message has suspicious elements.",
            "Scam":       "WARNING: This appears to be a scam! Do not click links or share personal info.",
        }
        return responses.get(risk_level, "Unable to analyze.")