app.py

import os
import csv
import re
import threading
from typing import Optional, List, Dict, Any, Tuple
from difflib import SequenceMatcher

import joblib
import numpy as np
import pandas as pd
from fastapi import FastAPI
from fastapi.responses import JSONResponse
from huggingface_hub import hf_hub_download
from pydantic import BaseModel
from urllib.parse import urlparse

try:
    import xgboost as xgb  # type: ignore
except Exception:
    xgb = None

# NLP libraries for Text Preprocessing (Module 2)
try:
    import nltk
    from nltk.tokenize import word_tokenize
    from nltk.corpus import stopwords
    from nltk.stem import PorterStemmer, WordNetLemmatizer
    from textblob import TextBlob
    
    # Download required NLTK data on startup
    for resource in ['punkt', 'stopwords', 'wordnet', 'omw-1.4']:
        try:
            nltk.data.find(f'tokenizers/{resource}' if resource == 'punkt' else f'corpora/{resource}')
        except LookupError:
            nltk.download(resource, quiet=True)
    
    NLTK_AVAILABLE = True
except Exception as e:
    print(f"[WARNING] NLP libraries not available: {e}")
    NLTK_AVAILABLE = False


# Environment defaults
os.environ.setdefault("HOME", "/data")
os.environ.setdefault("XDG_CACHE_HOME", "/data/.cache")
os.environ.setdefault("HF_HOME", "/data/.cache")
os.environ.setdefault("TRANSFORMERS_CACHE", "/data/.cache")
os.environ.setdefault("TORCH_HOME", "/data/.cache")


# Config
URL_REPO = os.environ.get("HF_URL_MODEL_ID", "Perth0603/Random-Forest-Model-for-PhishingDetection")
URL_REPO_TYPE = os.environ.get("HF_URL_REPO_TYPE", "model")
URL_FILENAME = os.environ.get("HF_URL_FILENAME", "rf_url_phishing_xgboost_bst.joblib")
CACHE_DIR = os.environ.get("HF_CACHE_DIR", "/data/.cache")
os.makedirs(CACHE_DIR, exist_ok=True)

URL_POSITIVE_CLASS_ENV = os.environ.get("URL_POSITIVE_CLASS", "").strip().upper()

BASE_DIR = os.path.dirname(__file__)
AUTOCALIB_PHISHY_CSV = os.environ.get("AUTOCALIB_PHISHY_CSV", os.path.join(BASE_DIR, "autocalib_phishy.csv"))
AUTOCALIB_LEGIT_CSV = os.environ.get("AUTOCALIB_LEGIT_CSV", os.path.join(BASE_DIR, "autocalib_legit.csv"))
KNOWN_HOSTS_CSV = os.environ.get("KNOWN_HOSTS_CSV", os.path.join(BASE_DIR, "known_hosts.csv"))

# Initialize NLP components
if NLTK_AVAILABLE:
    stemmer = PorterStemmer()
    lemmatizer = WordNetLemmatizer()
    stop_words = set(stopwords.words('english'))
    PHISHING_KEYWORDS = {
        'urgent', 'verify', 'suspended', 'locked', 'confirm', 'update',
        'click', 'prize', 'winner', 'congratulations', 'expire', 'act now',
        'account', 'security', 'password', 'credit card', 'bank', 'payment',
        'refund', 'tax', 'irs', 'social security', 'ssn', 'login', 'signin',
        'alert', 'warning', 'action required', 'unusual activity', 'compromised'
    }

# Consolidated lookalike characters dictionary
LOOKALIKE_CHARS = {
    # Cyrillic
    'а': 'a', 'е': 'e', 'о': 'o', 'р': 'p', 'с': 'c', 'х': 'x',
    'у': 'y', 'ч': '4', 'ы': 'b', 'ь': 'b', 'і': 'i', 'ї': 'yi',
    'ґ': 'g', 'ė': 'e', 'ń': 'n', 'ș': 's', 'ț': 't',
    # Greek
    'α': 'a', 'ο': 'o', 'ν': 'v', 'τ': 't', 'ρ': 'p',
    # Latin Extended
    'ɑ': 'a', 'ɢ': 'g', 'ᴅ': 'd', 'ɡ': 'g', 'ɪ': 'i',
    'ɴ': 'n', 'ᴘ': 'p', 'ᴠ': 'v', 'ᴡ': 'w', 'ɨ': 'i'
}

BRAND_NAMES = [
    "facebook", "linkedin", "paypal", "google", "amazon", "apple",
    "microsoft", "instagram", "netflix", "twitter", "whatsapp", "bank", "hsbc", "yahoo", "outlook"
]

SUSPICIOUS_KEYWORDS = ["login", "verify", "secure", "update", "bank", "pay", "account", "webscr"]
SUSPICIOUS_TLDS = {"tk", "ml", "ga", "cf", "gq", "xyz", "top", "buzz", "icu", "fit", "rest", "work", "click", "country", "zip", "ru", "kim", "support", "ltd"}


app = FastAPI(
    title="PhishWatch Pro API",
    version="3.1.0",
    description="Phishing detection with calibrated confidence scores (50-85% range)"
)


class PredictUrlPayload(BaseModel):
    url: str


class PreprocessTextPayload(BaseModel):
    text: str
    include_sentiment: bool = True
    include_stemming: bool = True
    include_lemmatization: bool = True
    remove_stopwords: bool = True


_url_bundle: Optional[Dict[str, Any]] = None
_url_lock = threading.Lock()
_URL_EXTRACT_RE = re.compile(r"(https?://[^\s<>\"'\)\]]+)", re.IGNORECASE)
_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://")


# ============================================================================
# UTILITY FUNCTIONS (Consolidated)
# ============================================================================

def _normalize_host(value: str) -> str:
    v = value.strip().lower()
    return v[4:] if v.startswith("www.") else v


def _host_matches_any(host: str, known: List[str]) -> bool:
    base = _normalize_host(host)
    for item in known:
        k = _normalize_host(item)
        if base == k or base.endswith("." + k):
            return True
    return False


def _sanitize_input_url(text: str) -> str:
    v = (text or "").strip()
    if v.startswith("@"):
        v = v.lstrip("@").strip()
    m = _URL_EXTRACT_RE.search(v)
    if m:
        v = m.group(1)
    return v.strip("<>[]()")


def _ensure_scheme(u: str) -> str:
    u = (u or "").strip()
    return u if _SCHEME_RE.match(u) else ("http://" + u)


def _normalize_url_string(url: str) -> str:
    return (url or "").strip().rstrip("/")


def _normalize_brand(s: str) -> str:
    return re.sub(r"[^a-z]", "", s.lower())


def _read_urls_from_csv(path: str) -> List[str]:
    urls: List[str] = []
    try:
        with open(path, newline="", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            if "url" in (reader.fieldnames or []):
                for row in reader:
                    val = str(row.get("url", "")).strip()
                    if val:
                        urls.append(val)
            else:
                f.seek(0)
                for row in csv.reader(f):
                    if row:
                        val = str(row[0]).strip()
                        if val.lower() != "url" and val:
                            urls.append(val)
    except FileNotFoundError:
        pass
    except Exception as e:
        print(f"[csv] failed reading URLs from {path}: {e}")
    return urls


def _read_hosts_from_csv(path: str) -> Dict[str, str]:
    out: Dict[str, str] = {}
    try:
        with open(path, newline="", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            fields = [x.lower() for x in (reader.fieldnames or [])]
            if "host" in fields and "label" in fields:
                for row in reader:
                    host = str(row.get("host", "")).strip()
                    label = str(row.get("label", "")).strip().upper()
                    if host and label in ("PHISH", "LEGIT"):
                        out[host] = label
    except FileNotFoundError:
        pass
    except Exception as e:
        print(f"[csv] failed reading hosts from {path}: {e}")
    return out


def _shannon_entropy(txt: str) -> float:
    if not txt:
        return 0.0
    counts: Dict[str, int] = {}
    for ch in txt:
        counts[ch] = counts.get(ch, 0) + 1
    total = float(len(txt))
    entropy = 0.0
    for n in counts.values():
        p = n / total
        entropy -= p * np.log2(p)
    return float(entropy)


def _detect_lookalike_chars(url: str) -> bool:
    """Check if URL contains homoglyph/lookalike characters"""
    for char in (url or ""):
        if char in LOOKALIKE_CHARS:
            return True
    return False


def _check_typosquat(url_str: str) -> bool:
    """Check for typosquatting patterns"""
    host = (urlparse(_ensure_scheme(url_str)).hostname or "").lower()
    sld = host.split(".")[-2] if "." in host else host
    clean_sld = _normalize_brand(sld)
    
    if not clean_sld:
        return False
    
    best_similarity = max(
        SequenceMatcher(None, clean_sld, _normalize_brand(b)).ratio()
        for b in BRAND_NAMES
    )
    
    has_digits = bool(re.search(r"\d", sld))
    has_hyphen = "-" in sld
    is_official = any(host.endswith(f"{_normalize_brand(b)}.com") for b in BRAND_NAMES)
    
    return (best_similarity >= 0.90) and (has_digits or has_hyphen) and (not is_official)


def _count_suspicious_features(url_str: str) -> Tuple[int, List[str]]:
    """Count suspicious indicators in URL"""
    count = 0
    features = []
    
    # Suspicious keywords
    for kw in SUSPICIOUS_KEYWORDS:
        if kw in url_str.lower():
            count += 1
            features.append(f"keyword:{kw}")
    
    # IP address
    if re.search(r"(?:\d{1,3}\.){3}\d{1,3}", url_str):
        count += 1
        features.append("ip_address")
    
    # Excessive length
    if len(url_str) > 75:
        count += 1
        features.append("long_url")
    
    # Many subdomains
    host = (urlparse(_ensure_scheme(url_str)).hostname or "").lower()
    if host.count('.') > 3:
        count += 1
        features.append("many_subdomains")
    
    return count, features


def _calibrate_confidence(
    is_phishing: bool,
    raw_proba: float,
    url_str: str,
    detection_method: str
) -> Dict[str, Any]:
    """
    Universal confidence calibration function.
    Returns scores in 50-85% range for both phishing and legitimate URLs.
    """
    
    # === PHISHING DETECTION ===
    if is_phishing:
        if detection_method == "lookalike":
            # Lookalike: 68-78%
            calibrated = 0.68 + (min(raw_proba, 1.0) * 0.10)
            return {
                "calibrated_proba": float(calibrated),
                "confidence_level": "MEDIUM-HIGH",
                "detection_method": "Homoglyph/Lookalike Character",
                "explanation": "URL contains visually deceptive characters (e.g., Cyrillic 'а' vs ASCII 'a')"
            }
        
        elif detection_method == "typosquat":
            # Typosquatting: 63-75%
            calibrated = 0.63 + (min(raw_proba, 1.0) * 0.12)
            return {
                "calibrated_proba": float(calibrated),
                "confidence_level": "MEDIUM",
                "detection_method": "Brand Typosquatting",
                "explanation": "Domain mimics a popular brand with suspicious modifications"
            }
        
        elif detection_method == "csv_match":
            # Known phishing URL: 78-85%
            calibrated = 0.78 + (min(raw_proba, 1.0) * 0.07)
            return {
                "calibrated_proba": float(calibrated),
                "confidence_level": "HIGH",
                "detection_method": "Known Phishing Database",
                "explanation": "URL matches verified phishing database"
            }
        
        elif detection_method == "host_match":
            # Known malicious host: 75-83%
            calibrated = 0.75 + (min(raw_proba, 1.0) * 0.08)
            return {
                "calibrated_proba": float(calibrated),
                "confidence_level": "HIGH",
                "detection_method": "Malicious Host Database",
                "explanation": "Domain listed in malicious hosts database"
            }
        
        else:  # ML model detection
            susp_count, susp_features = _count_suspicious_features(url_str)
            
            if raw_proba >= 0.90 and susp_count >= 3:
                # Very confident + multiple indicators: 78-85%
                calibrated = 0.78 + (min(raw_proba, 1.0) * 0.07)
                confidence = "HIGH"
            elif raw_proba >= 0.75:
                # Medium-high confidence: 70-80%
                calibrated = 0.70 + (min(raw_proba, 1.0) * 0.10)
                confidence = "MEDIUM-HIGH"
            elif raw_proba >= 0.60:
                # Medium confidence: 62-75%
                calibrated = 0.62 + (min(raw_proba, 1.0) * 0.13)
                confidence = "MEDIUM"
            else:
                # Lower confidence: 55-68%
                calibrated = 0.55 + (min(raw_proba, 1.0) * 0.13)
                confidence = "LOW-MEDIUM"
            
            feature_text = f" ({susp_count} indicators: {', '.join(susp_features[:3])})" if susp_features else ""
            return {
                "calibrated_proba": float(calibrated),
                "confidence_level": confidence,
                "detection_method": f"ML Analysis{feature_text}",
                "explanation": "Random Forest model detected phishing patterns in URL structure"
            }
    
    # === LEGITIMATE DETECTION ===
    else:
        if detection_method in ["csv_match", "host_match"]:
            # Known legitimate: 70-80%
            calibrated = 0.70 + (min(1.0 - raw_proba, 1.0) * 0.10)
            return {
                "calibrated_proba": float(calibrated),
                "confidence_level": "HIGH",
                "detection_method": "Verified Legitimate Database",
                "explanation": "URL verified as legitimate in trusted database"
            }
        else:
            # ML model says legitimate: 72-82%
            legit_confidence = 1.0 - raw_proba
            calibrated = 0.72 + (min(legit_confidence, 1.0) * 0.10)
            return {
                "calibrated_proba": float(calibrated),
                "confidence_level": "HIGH" if legit_confidence > 0.8 else "MEDIUM-HIGH",
                "detection_method": "ML Analysis",
                "explanation": "Random Forest model detected legitimate URL patterns"
            }


def _engineer_features(urls: List[str], feature_cols: List[str]) -> pd.DataFrame:
    """Feature engineering matching notebook implementation"""
    s = pd.Series(urls, dtype=str)
    out = pd.DataFrame()

    # Basic features
    out["url_len"] = s.str.len().fillna(0)
    out["count_dot"] = s.str.count(r"\.")
    out["count_hyphen"] = s.str.count("-")
    out["count_digit"] = s.str.count(r"\d")
    out["count_at"] = s.str.count("@")
    out["count_qmark"] = s.str.count(r"\?")
    out["count_eq"] = s.str.count("=")
    out["count_slash"] = s.str.count("/")
    out["digit_ratio"] = (out["count_digit"] / out["url_len"].replace(0, np.nan)).fillna(0)
    out["has_ip"] = s.str.contains(r"(?:\d{1,3}\.){3}\d{1,3}").astype(int)
    
    for tok in SUSPICIOUS_KEYWORDS:
        out[f"has_{tok}"] = s.str.contains(tok, case=False, regex=False).astype(int)
    
    out["starts_https"] = s.str.startswith("https").astype(int)
    out["ends_with_exe"] = s.str.endswith(".exe").astype(int)
    out["ends_with_zip"] = s.str.endswith(".zip").astype(int)

    # Host/domain features
    hosts = s.apply(lambda x: (urlparse(_ensure_scheme(x)).hostname or "").lower())
    out["host_len"] = hosts.str.len().fillna(0)
    
    label_counts = hosts.str.count(r"\.") + 1
    out["subdomain_count"] = (label_counts - 2).clip(lower=0).fillna(0)

    parts_series = hosts.str.split(".")
    tld_series = parts_series.apply(lambda p: p[-1] if len(p) >= 1 else "")
    sld_series = parts_series.apply(lambda p: p[-2] if len(p) >= 2 else "")

    out["tld_suspicious"] = tld_series.apply(lambda t: 1 if t.lower() in SUSPICIOUS_TLDS else 0)
    out["has_punycode"] = hosts.str.contains("xn--").astype(int)
    out["sld_len"] = sld_series.str.len().fillna(0)
    
    def _ratio_digits(txt: str) -> float:
        if not txt:
            return 0.0
        digits = sum(c.isdigit() for c in txt)
        return float(digits) / float(len(txt))
    
    out["sld_digit_ratio"] = sld_series.apply(_ratio_digits)
    out["sld_entropy"] = sld_series.apply(_shannon_entropy)

    # Brand similarity
    def _max_brand_similarity(host: str) -> float:
        if not host:
            return 0.0
        sld = host.split(".")[-2] if "." in host else host
        similarities = []
        for brand in BRAND_NAMES:
            similarities.append(SequenceMatcher(None, host, brand).ratio())
            similarities.append(SequenceMatcher(None, sld, brand).ratio())
        return max(similarities) if similarities else 0.0

    out["max_brand_sim"] = hosts.apply(_max_brand_similarity)
    out["like_facebook"] = hosts.apply(
        lambda h: 1 if SequenceMatcher(None, h.split(".")[-2] if "." in h else h, "facebook").ratio() >= 0.82 else 0
    )
    out["has_lookalike_chars"] = s.apply(lambda u: 1 if _detect_lookalike_chars(u) else 0)

    return out.reindex(columns=feature_cols, fill_value=0)


def _load_url_model():
    global _url_bundle
    if _url_bundle is None:
        with _url_lock:
            if _url_bundle is None:
                local_path = os.path.join(os.getcwd(), URL_FILENAME)
                if os.path.exists(local_path):
                    _url_bundle = joblib.load(local_path)
                else:
                    model_path = hf_hub_download(
                        repo_id=URL_REPO,
                        filename=URL_FILENAME,
                        repo_type=URL_REPO_TYPE,
                        cache_dir=CACHE_DIR,
                    )
                    _url_bundle = joblib.load(model_path)


# ============================================================================
# API ENDPOINTS
# ============================================================================

@app.get("/")
def root():
    return {
        "status": "ok",
        "service": "PhishWatch Pro API",
        "version": "3.1.0",
        "modules": {
            "module_2_text_preprocessing": NLTK_AVAILABLE,
            "module_4_url_analyzer": True
        },
        "confidence_range": "50-85% (calibrated for both phishing and legitimate)"
    }


@app.post("/preprocess-text")
def preprocess_text(payload: PreprocessTextPayload):
    """Module 2: Text Preprocessing with calibrated confidence (50-85%)"""
    if not NLTK_AVAILABLE:
        return JSONResponse(
            status_code=503,
            content={"error": "NLP libraries not available. Install: pip install nltk textblob"}
        )
    
    try:
        text = (payload.text or "").strip()
        if not text:
            return JSONResponse(status_code=400, content={"error": "Empty text"})

        tokens = word_tokenize(text.lower())
        tokens_filtered = [
            t for t in tokens 
            if t.isalnum() and (not payload.remove_stopwords or t not in stop_words)
        ]

        stemmed_tokens = [stemmer.stem(t) for t in tokens_filtered] if payload.include_stemming else []
        lemmatized_tokens = [lemmatizer.lemmatize(t) for t in tokens_filtered] if payload.include_lemmatization else []

        sentiment_data = {}
        phishing_indicators = {}
        
        if payload.include_sentiment:
            blob = TextBlob(text)
            sentiment_data = {
                "polarity": float(blob.sentiment.polarity),
                "subjectivity": float(blob.sentiment.subjectivity),
                "classification": (
                    "positive" if blob.sentiment.polarity > 0.1 else
                    "negative" if blob.sentiment.polarity < -0.1 else "neutral"
                )
            }

            text_lower = text.lower()
            detected_keywords = [kw for kw in PHISHING_KEYWORDS if kw in text_lower]
            keyword_density = len(detected_keywords) / max(len(tokens_filtered), 1)
            
            urgency_detected = any(
                kw in detected_keywords 
                for kw in ['urgent', 'expire', 'act now', 'suspended', 'locked', 'warning', 'alert']
            )
            emotional_appeal = blob.sentiment.subjectivity > 0.6
            
            # Calibrated confidence: 50-82%
            base_score = 0.50 + (len(detected_keywords) * 0.08) + (keyword_density * 0.15)
            if urgency_detected:
                base_score += 0.12
            if emotional_appeal:
                base_score += 0.08
            base_score = min(0.82, base_score)
            
            phishing_indicators = {
                "suspicious_keywords": detected_keywords,
                "keyword_count": len(detected_keywords),
                "keyword_density": float(keyword_density),
                "urgency_detected": urgency_detected,
                "emotional_appeal": emotional_appeal,
                "risk_score": float(base_score),
                "confidence_level": (
                    "HIGH" if base_score >= 0.72 else
                    "MEDIUM" if base_score >= 0.58 else "LOW"
                ),
                "risk_level": (
                    "HIGH" if len(detected_keywords) >= 3 or urgency_detected else
                    "MEDIUM" if len(detected_keywords) >= 1 else "LOW"
                )
            }

        return {
            "module": "text_preprocessing",
            "original_text": text,
            "tokens": tokens[:100],
            "token_count": len(tokens),
            "filtered_tokens": tokens_filtered[:100],
            "filtered_token_count": len(tokens_filtered),
            "cleaned_text": " ".join(tokens_filtered),
            "stemmed_text": " ".join(stemmed_tokens) if stemmed_tokens else None,
            "lemmatized_text": " ".join(lemmatized_tokens) if lemmatized_tokens else None,
            "sentiment": sentiment_data if sentiment_data else None,
            "phishing_indicators": phishing_indicators if phishing_indicators else None,
            "preprocessing_applied": {
                "tokenization": True,
                "stopword_removal": payload.remove_stopwords,
                "stemming": payload.include_stemming,
                "lemmatization": payload.include_lemmatization,
                "sentiment_analysis": payload.include_sentiment
            }
        }

    except Exception as e:
        return JSONResponse(status_code=500, content={"error": str(e)})


@app.post("/predict-url")
def predict_url(payload: PredictUrlPayload):
    """Module 4: URL Analyzer with calibrated confidence (both phishing and legit: 50-85%)"""
    try:
        _load_url_model()

        phishy_list = _read_urls_from_csv(AUTOCALIB_PHISHY_CSV)
        legit_list = _read_urls_from_csv(AUTOCALIB_LEGIT_CSV)
        host_map = _read_hosts_from_csv(KNOWN_HOSTS_CSV)

        bundle = _url_bundle
        if not isinstance(bundle, dict) or "model" not in bundle:
            raise RuntimeError("Invalid model bundle")

        model = bundle["model"]
        feature_cols: List[str] = bundle.get("feature_cols") or []
        url_col: str = bundle.get("url_col") or "url"
        model_type: str = bundle.get("model_type") or ""

        raw_input = (payload.url or "").strip()
        url_str = _sanitize_input_url(raw_input)
        if not url_str:
            return JSONResponse(status_code=400, content={"error": "Empty url"})

        phish_is_positive = True if URL_POSITIVE_CLASS_ENV == "" else (URL_POSITIVE_CLASS_ENV == "PHISH")
        norm_url = _normalize_url_string(url_str)
        phishy_set = {_normalize_url_string(u) for u in phishy_list}
        legit_set = {_normalize_url_string(u) for u in legit_list}

        # CSV match
        if norm_url in phishy_set or norm_url in legit_set:
            is_phishing = norm_url in phishy_set
            raw_proba = 0.99 if is_phishing else 0.01
            calibration = _calibrate_confidence(is_phishing, raw_proba, url_str, "csv_match")
            
            label = "PHISH" if is_phishing else "LEGIT"
            phish_proba = calibration["calibrated_proba"] if is_phishing else (1.0 - calibration["calibrated_proba"])
            predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
            score = phish_proba if is_phishing else calibration["calibrated_proba"]
            
            return {
                "module": "url_analyzer",
                "label": label,
                "predicted_label": int(predicted_label),
                "score": float(score),
                "phishing_probability": float(phish_proba) if is_phishing else float(1.0 - score),
                "confidence_level": calibration["confidence_level"],
                "detection_method": calibration["detection_method"],
                "explanation": calibration["explanation"],
                "backend": str(model_type),
                "threshold": 0.5,
                "url_col": url_col,
            }

        # Host match
        host = (urlparse(_ensure_scheme(url_str)).hostname or "").lower()
        if host and host_map:
            for h, lbl in host_map.items():
                if _host_matches_any(host, [h]):
                    is_phishing = (lbl == "PHISH")
                    raw_proba = 0.99 if is_phishing else 0.01
                    calibration = _calibrate_confidence(is_phishing, raw_proba, url_str, "host_match")
                    
                    label = lbl
                    phish_proba = calibration["calibrated_proba"] if is_phishing else (1.0 - calibration["calibrated_proba"])
                    predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
                    score = phish_proba if is_phishing else calibration["calibrated_proba"]
                    
                    return {
                        "module": "url_analyzer",
                        "label": label,
                        "predicted_label": int(predicted_label),
                        "score": float(score),
                        "phishing_probability": float(phish_proba) if is_phishing else float(1.0 - score),
                        "confidence_level": calibration["confidence_level"],
                        "detection_method": calibration["detection_method"],
                        "explanation": calibration["explanation"],
                        "backend": str(model_type),
                        "threshold": 0.5,
                        "url_col": url_col,
                    }

        # Lookalike check
        if _detect_lookalike_chars(url_str):
            calibration = _calibrate_confidence(True, 0.95, url_str, "lookalike")
            return {
                "module": "url_analyzer",
                "label": "PHISH",
                "predicted_label": 1 if phish_is_positive else 0,
                "score": float(calibration["calibrated_proba"]),
                "phishing_probability": float(calibration["calibrated_proba"]),
                "confidence_level": calibration["confidence_level"],
                "detection_method": calibration["detection_method"],
                "explanation": calibration["explanation"],
                "backend": "heuristic",
                "threshold": 0.5,
                "url_col": url_col,
            }

        # Typosquat check
        if _check_typosquat(url_str):
            calibration = _calibrate_confidence(True, 0.90, url_str, "typosquat")
            return {
                "module": "url_analyzer",
                "label": "PHISH",
                "predicted_label": 1 if phish_is_positive else 0,
                "score": float(calibration["calibrated_proba"]),
                "phishing_probability": float(calibration["calibrated_proba"]),
                "confidence_level": calibration["confidence_level"],
                "detection_method": calibration["detection_method"],
                "explanation": calibration["explanation"],
                "backend": "heuristic",
                "threshold": 0.5,
                "url_col": url_col,
            }

        # ML model inference
        feats = _engineer_features([url_str], feature_cols)
        
        if model_type == "xgboost_bst":
            if xgb is None:
                raise RuntimeError("xgboost not installed")
            dmat = xgb.DMatrix(feats)
            raw_p_class1 = float(model.predict(dmat)[0])
        elif hasattr(model, "predict_proba"):
            raw_p_class1 = float(model.predict_proba(feats)[:, 1][0])
        else:
            pred = model.predict(feats)[0]
            raw_p_class1 = 1.0 if int(pred) == 1 else 0.0

        raw_phish_proba = raw_p_class1 if phish_is_positive else (1.0 - raw_p_class1)
        is_phishing = raw_phish_proba >= 0.5
        
        calibration = _calibrate_confidence(is_phishing, raw_phish_proba, url_str, "ml_model")
        
        label = "PHISH" if is_phishing else "LEGIT"
        predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
        
        if is_phishing:
            phish_proba = calibration["calibrated_proba"]
            score = phish_proba
        else:
            legit_confidence = calibration["calibrated_proba"]
            phish_proba = 1.0 - legit_confidence
            score = legit_confidence

        return {
            "module": "url_analyzer",
            "label": label,
            "predicted_label": int(predicted_label),
            "score": float(score),
            "phishing_probability": float(phish_proba),
            "confidence_level": calibration["confidence_level"],
            "detection_method": calibration["detection_method"],
            "explanation": calibration["explanation"],
            "backend": str(model_type),
            "threshold": 0.5,
            "url_col": url_col,
        }
        
    except Exception as e:
        return JSONResponse(status_code=500, content={"error": str(e)})