Update app.py

app.py

@@ -343,6 +343,128 @@ def _normalize_url_string(url: str) -> str:
     return (url or "").strip().rstrip("/")
 
 
+def _calibrate_confidence(raw_proba: float, url_str: str, detection_reason: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Calibrate confidence score based on detection method to avoid showing 100% confidence.
+    
+    Returns dict with:
+    - calibrated_proba: adjusted probability (50-85% for heuristic detections, 60-95% for ML)
+    - confidence_level: HIGH/MEDIUM/LOW
+    - detection_method: what triggered the detection
+    """
+    
+    # Heuristic-based detections get lower confidence (50-75%)
+    if detection_reason == "lookalike_character_detected":
+        # Lookalike characters: 70-80% confidence
+        calibrated = 0.70 + (raw_proba * 0.10)
+        return {
+            "calibrated_proba": float(calibrated),
+            "confidence_level": "MEDIUM-HIGH",
+            "detection_method": "Homoglyph/Lookalike Character Pattern",
+            "explanation": "URL contains characters that visually resemble legitimate letters (e.g., Cyrillic 'а' instead of 'a')"
+        }
+    
+    elif detection_reason == "typosquat_guard":
+        # Typosquatting: 65-78% confidence
+        calibrated = 0.65 + (raw_proba * 0.13)
+        return {
+            "calibrated_proba": float(calibrated),
+            "confidence_level": "MEDIUM",
+            "detection_method": "Brand Typosquatting Pattern",
+            "explanation": "Domain name closely resembles a popular brand with suspicious modifications (digits/hyphens)"
+        }
+    
+    elif detection_reason == "csv_url_match":
+        # Known phishing URL from CSV: 85-95% confidence
+        calibrated = 0.85 + (raw_proba * 0.10)
+        return {
+            "calibrated_proba": float(calibrated),
+            "confidence_level": "HIGH",
+            "detection_method": "Known Phishing URL Database Match",
+            "explanation": "URL matches a verified phishing URL in our database"
+        }
+    
+    elif detection_reason == "known_host_match":
+        # Known host from CSV: 80-92% confidence
+        calibrated = 0.80 + (raw_proba * 0.12)
+        return {
+            "calibrated_proba": float(calibrated),
+            "confidence_level": "HIGH",
+            "detection_method": "Known Malicious Host Database Match",
+            "explanation": "Domain is listed in our verified malicious hosts database"
+        }
+    
+    # ML model detections: calibrate based on raw probability
+    else:
+        # Parse URL for additional context
+        host = (urlparse(_ensure_scheme(url_str)).hostname or "").lower()
+        
+        # Count suspicious indicators
+        suspicious_count = 0
+        suspicious_features = []
+        
+        # Check for suspicious keywords
+        suspicious_keywords = ["login", "verify", "secure", "update", "bank", "pay", "account", "webscr"]
+        for kw in suspicious_keywords:
+            if kw in url_str.lower():
+                suspicious_count += 1
+                suspicious_features.append(f"keyword:{kw}")
+        
+        # Check for IP address
+        if re.search(r"(?:\d{1,3}\.){3}\d{1,3}", url_str):
+            suspicious_count += 1
+            suspicious_features.append("ip_address")
+        
+        # Check for excessive length
+        if len(url_str) > 75:
+            suspicious_count += 1
+            suspicious_features.append("excessive_length")
+        
+        # Check for many subdomains
+        if host.count('.') > 3:
+            suspicious_count += 1
+            suspicious_features.append("many_subdomains")
+        
+        # Calibrate based on ML confidence and feature count
+        if raw_proba >= 0.90:
+            # Very high ML confidence + multiple indicators: 82-92%
+            if suspicious_count >= 3:
+                calibrated = 0.82 + (raw_proba * 0.10)
+                confidence = "HIGH"
+            # High ML confidence with fewer indicators: 75-88%
+            else:
+                calibrated = 0.75 + (raw_proba * 0.13)
+                confidence = "MEDIUM-HIGH"
+        
+        elif raw_proba >= 0.75:
+            # Medium-high ML confidence: 68-82%
+            if suspicious_count >= 2:
+                calibrated = 0.68 + (raw_proba * 0.14)
+                confidence = "MEDIUM-HIGH"
+            else:
+                calibrated = 0.60 + (raw_proba * 0.15)
+                confidence = "MEDIUM"
+        
+        elif raw_proba >= 0.60:
+            # Medium ML confidence: 55-72%
+            calibrated = 0.55 + (raw_proba * 0.17)
+            confidence = "MEDIUM"
+        
+        else:
+            # Lower confidence: keep closer to original but cap at 65%
+            calibrated = min(0.65, 0.50 + (raw_proba * 0.15))
+            confidence = "LOW-MEDIUM"
+        
+        feature_text = f" (Detected: {', '.join(suspicious_features[:3])})" if suspicious_features else ""
+        
+        return {
+            "calibrated_proba": float(calibrated),
+            "confidence_level": confidence,
+            "detection_method": f"Machine Learning Analysis ({suspicious_count} suspicious indicators){feature_text}",
+            "explanation": "Random Forest model detected multiple phishing patterns in URL structure and content"
+        }
+
+
 # ============================================================================
 # API ENDPOINTS
 # ============================================================================
@@ -433,6 +555,13 @@ def preprocess_text(payload: PreprocessTextPayload):
             ])
             emotional_appeal = blob.sentiment.subjectivity > 0.6
             
+            # Calibrated confidence for text analysis (50-85%)
+            base_score = min(0.85, 0.50 + (len(detected_keywords) * 0.10) + (keyword_density * 0.25))
+            if urgency_detected:
+                base_score = min(0.85, base_score + 0.15)
+            if emotional_appeal:
+                base_score = min(0.85, base_score + 0.10)
+            
             phishing_indicators = {
                 "suspicious_keywords": detected_keywords,
                 "keyword_count": len(detected_keywords),
@@ -440,11 +569,11 @@ def preprocess_text(payload: PreprocessTextPayload):
                 "urgency_detected": urgency_detected,
                 "emotional_appeal": emotional_appeal,
                 "high_subjectivity": blob.sentiment.subjectivity > 0.6,
-                "risk_score": min(1.0, 
-                    len(detected_keywords) * 0.12 + 
-                    (0.25 if urgency_detected else 0) +
-                    (0.20 if emotional_appeal else 0) +
-                    (keyword_density * 0.3)
+                "risk_score": float(base_score),
+                "confidence_level": (
+                    "HIGH" if base_score >= 0.75 else
+                    "MEDIUM" if base_score >= 0.60 else
+                    "LOW"
                 ),
                 "risk_level": (
                     "HIGH" if len(detected_keywords) >= 3 or urgency_detected else
@@ -488,12 +617,8 @@ def predict_url(payload: PredictUrlPayload):
     """
     Module 4: URL Analyzer
     
-    Analyzes URLs for phishing using Random Forest model with:
-    - Structural analysis (length, symbols, patterns)
-    - Domain analysis (SLD, TLD, subdomains)
-    - Typosquatting detection
-    - Lookalike character detection
-    - Brand similarity analysis
+    Analyzes URLs for phishing using Random Forest model with calibrated confidence scores.
+    Confidence ranges: 50-85% (heuristic), 60-92% (ML model)
     """
     try:
         _load_url_model()
@@ -517,27 +642,43 @@ def predict_url(payload: PredictUrlPayload):
         if not url_str:
             return JSONResponse(status_code=400, content={"error": "Empty url"})
 
+        phish_is_positive = True if URL_POSITIVE_CLASS_ENV == "" else (URL_POSITIVE_CLASS_ENV == "PHISH")
+
         # URL-level override via CSV lists
         norm_url = _normalize_url_string(url_str)
         phishy_set = { _normalize_url_string(u) for u in phishy_list }
         legit_set = { _normalize_url_string(u) for u in legit_list }
 
         if norm_url in phishy_set or norm_url in legit_set:
-            phish_is_positive = True if URL_POSITIVE_CLASS_ENV == "" else (URL_POSITIVE_CLASS_ENV == "PHISH")
             label = "PHISH" if norm_url in phishy_set else "LEGIT"
+            raw_proba = 0.99 if label == "PHISH" else 0.01
+            
+            if label == "PHISH":
+                calibration = _calibrate_confidence(raw_proba, url_str, "csv_url_match")
+                phish_proba = calibration["calibrated_proba"]
+            else:
+                phish_proba = raw_proba
+                calibration = {
+                    "confidence_level": "HIGH",
+                    "detection_method": "Known Legitimate URL",
+                    "explanation": "URL verified as legitimate in our database"
+                }
+            
             predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
-            phish_proba = 0.99 if label == "PHISH" else 0.01
             score = phish_proba if label == "PHISH" else (1.0 - phish_proba)
+            
             return {
                 "module": "url_analyzer",
                 "label": label,
                 "predicted_label": int(predicted_label),
                 "score": float(score),
                 "phishing_probability": float(phish_proba),
+                "confidence_level": calibration["confidence_level"],
+                "detection_method": calibration["detection_method"],
+                "explanation": calibration["explanation"],
                 "backend": str(model_type),
                 "threshold": 0.5,
                 "url_col": url_col,
-                "override": {"reason": "csv_url_match"},
             }
 
         # Known-host override
@@ -545,21 +686,35 @@ def predict_url(payload: PredictUrlPayload):
         if host and host_map:
             for h, lbl in host_map.items():
                 if _host_matches_any(host, [h]):
-                    phish_is_positive = True if URL_POSITIVE_CLASS_ENV == "" else (URL_POSITIVE_CLASS_ENV == "PHISH")
                     label = lbl
+                    raw_proba = 0.99 if label == "PHISH" else 0.01
+                    
+                    if label == "PHISH":
+                        calibration = _calibrate_confidence(raw_proba, url_str, "known_host_match")
+                        phish_proba = calibration["calibrated_proba"]
+                    else:
+                        phish_proba = raw_proba
+                        calibration = {
+                            "confidence_level": "HIGH",
+                            "detection_method": "Known Legitimate Host",
+                            "explanation": "Domain verified as legitimate"
+                        }
+                    
                     predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
-                    phish_proba = 0.99 if label == "PHISH" else 0.01
                     score = phish_proba if label == "PHISH" else (1.0 - phish_proba)
+                    
                     return {
                         "module": "url_analyzer",
                         "label": label,
                         "predicted_label": int(predicted_label),
                         "score": float(score),
                         "phishing_probability": float(phish_proba),
+                        "confidence_level": calibration["confidence_level"],
+                        "detection_method": calibration["detection_method"],
+                        "explanation": calibration["explanation"],
                         "backend": str(model_type),
                         "threshold": 0.5,
                         "url_col": url_col,
-                        "override": {"reason": "known_host_match"},
                     }
 
         # Lookalike character guard
@@ -579,21 +734,26 @@ def predict_url(payload: PredictUrlPayload):
         
         for char in url_str:
             if char in all_lookalikes:
-                phish_is_positive = True if URL_POSITIVE_CLASS_ENV == "" else (URL_POSITIVE_CLASS_ENV == "PHISH")
                 label = "PHISH"
+                raw_proba = 0.95
+                calibration = _calibrate_confidence(raw_proba, url_str, "lookalike_character_detected")
+                phish_proba = calibration["calibrated_proba"]
+                
                 predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
-                phish_proba = 0.95
                 score = phish_proba
+                
                 return {
                     "module": "url_analyzer",
                     "label": label,
                     "predicted_label": int(predicted_label),
                     "score": float(score),
                     "phishing_probability": float(phish_proba),
+                    "confidence_level": calibration["confidence_level"],
+                    "detection_method": calibration["detection_method"],
+                    "explanation": calibration["explanation"],
                     "backend": "lookalike_guard",
                     "threshold": 0.5,
                     "url_col": url_col,
-                    "rule": "lookalike_character_detected",
                 }
 
         # Typosquat guard
@@ -614,22 +774,28 @@ def predict_url(payload: PredictUrlPayload):
             has_digits = bool(re.search(r"\d", s_sld))
             has_hyphen = ("-" in s_sld)
             is_official = any(s_host.endswith(f"{_normalize_brand(b)}.com") for b in brands)
+            
             if (best >= 0.90) and (has_digits or has_hyphen) and (not is_official):
-                phish_is_positive = True if URL_POSITIVE_CLASS_ENV == "" else (URL_POSITIVE_CLASS_ENV == "PHISH")
                 label = "PHISH"
+                raw_proba = 0.90
+                calibration = _calibrate_confidence(raw_proba, url_str, "typosquat_guard")
+                phish_proba = calibration["calibrated_proba"]
+                
                 predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
-                phish_proba = 0.90
                 score = phish_proba
+                
                 return {
                     "module": "url_analyzer",
                     "label": label,
                     "predicted_label": int(predicted_label),
                     "score": float(score),
                     "phishing_probability": float(phish_proba),
+                    "confidence_level": calibration["confidence_level"],
+                    "detection_method": calibration["detection_method"],
+                    "explanation": calibration["explanation"],
                     "backend": "typosquat_guard",
                     "threshold": 0.5,
                     "url_col": url_col,
-                    "rule": "typosquat_guard",
                 }
 
         # ML model inference
@@ -645,8 +811,20 @@ def predict_url(payload: PredictUrlPayload):
             pred = model.predict(feats)[0]
             raw_p_class1 = 1.0 if int(pred) == 1 else 0.0
 
-        phish_is_positive = True if URL_POSITIVE_CLASS_ENV == "" else (URL_POSITIVE_CLASS_ENV == "PHISH")
-        phish_proba = raw_p_class1 if phish_is_positive else (1.0 - raw_p_class1)
+        raw_phish_proba = raw_p_class1 if phish_is_positive else (1.0 - raw_p_class1)
+        
+        # Calibrate ML model predictions
+        if raw_phish_proba >= 0.5:
+            calibration = _calibrate_confidence(raw_phish_proba, url_str, None)
+            phish_proba = calibration["calibrated_proba"]
+        else:
+            phish_proba = raw_phish_proba
+            calibration = {
+                "confidence_level": "HIGH",
+                "detection_method": "Machine Learning Analysis",
+                "explanation": "Random Forest model analysis indicates legitimate URL patterns"
+            }
+        
         label = "PHISH" if phish_proba >= 0.5 else "LEGIT"
         predicted_label = 1 if ((label == "PHISH") == phish_is_positive) else 0
         score = phish_proba if label == "PHISH" else (1.0 - phish_proba)
@@ -657,6 +835,9 @@ def predict_url(payload: PredictUrlPayload):
             "predicted_label": int(predicted_label),
             "score": float(score),
             "phishing_probability": float(phish_proba),
+            "confidence_level": calibration.get("confidence_level", "MEDIUM"),
+            "detection_method": calibration.get("detection_method", "ML Analysis"),
+            "explanation": calibration.get("explanation", "Statistical analysis of URL patterns"),
             "backend": str(model_type),
             "threshold": 0.5,
             "url_col": url_col,