update API endpoints

.gitignore

@@ -0,0 +1,68 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+venv/
+env/
+ENV/
+env.bak/
+venv.bak/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# Logs
+*.log
+logs/
+
+# Cache
+.cache/
+*.cache
+.pytest_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+__pycache__/
+*.pkl
+*.joblib
+.vscode/settings.json
+
+# Temporary files
+*.tmp
+*.temp
+*.bak

backend/.env

@@ -1,4 +1,5 @@
-VIRUSTOTAL_API_KEY=your_virustotal_key_here
-GOOGLE_SAFE_BROWSING_KEY=your_google_key_here
+VIRUSTOTAL_API_KEY=bb70b1af05e004605b7667a9eab263e36a08048ec854a7f1c030a958094e58a5
+SAFE_BROWSING_API_KEY=AIzaSyCpElL0PeB1Llkq_Jo58k-gjejwNIk5B4k
 APP_SECRET=phishguard_secret_2024
-DEBUG=True
+DEBUG=True
+ALLOWED_ORIGINS=https://rohanv56-phishing-detection-api.hf.space,http://localhost:8000

backend/ml_engine.py

@@ -83,11 +83,13 @@ class MLEngine:
     def load_models(self) -> None:
         errors = []
 
-        # URL model
+        # URL model - use joblib to load (consistent with joblib.dump in training)
         try:
-            with open(URL_MODEL_PATH, "rb") as f:
-                self.url_model = pickle.load(f)
+            self.url_model = joblib.load(URL_MODEL_PATH)
             logger.info(f"URL model loaded from {URL_MODEL_PATH}")
+            # Verify that it's a valid sklearn model
+            if not hasattr(self.url_model, 'predict') or not hasattr(self.url_model, 'predict_proba'):
+                raise TypeError(f"URL model is not a valid sklearn model. Type: {type(self.url_model)}")
         except FileNotFoundError:
             errors.append(f"URL model not found at {URL_MODEL_PATH}")
         except Exception as e:
@@ -95,8 +97,7 @@ class MLEngine:
 
         # URL feature columns
         try:
-            with open(URL_COLS_PATH, "rb") as f:
-                self.url_feature_cols = pickle.load(f)
+            self.url_feature_cols = joblib.load(URL_COLS_PATH)
             logger.info(f"URL feature cols loaded ({len(self.url_feature_cols)} features)")
         except FileNotFoundError:
             # Fall back to the canonical 30-feature list defined at top of file
@@ -109,6 +110,9 @@ class MLEngine:
         try:
             self.sms_model = joblib.load(SMS_MODEL_PATH)
             logger.info(f"SMS model loaded from {SMS_MODEL_PATH}")
+            # Verify it's a valid sklearn model/pipeline
+            if not hasattr(self.sms_model, 'predict') or not hasattr(self.sms_model, 'predict_proba'):
+                raise TypeError(f"SMS model is not a valid sklearn model. Type: {type(self.sms_model)}")
         except FileNotFoundError:
             errors.append(f"SMS model not found at {SMS_MODEL_PATH}")
         except Exception as e:

backend/models/schemas.py

@@ -16,6 +16,7 @@ class ThreatLevel(str, Enum):
 
 class URLScanRequest(BaseModel):
     url: str = Field(..., description="The URL to scan", example="http://example.com/login")
+    features: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Pre-extracted URL features (30 binary features). If not provided, defaults to empty dict.")
 
 class MLURLResult(BaseModel):
     prediction: str          # "phishing" | "legitimate"

backend/routes/scan.py

@@ -149,26 +149,6 @@ async def scan_url(request: URLScanRequest):
     )
 
 
-# Extend schema to include features field
-from pydantic import BaseModel
-from typing import Optional
-
-class URLScanRequestFull(URLScanRequest):
-    features: Optional[Dict[str, Any]] = {}
-
-# Override the endpoint to use the extended model
-router.routes = [r for r in router.routes if getattr(r, "path", "") != "/scan-url"]
-
-@router.post(
-    "/scan-url",
-    response_model=URLScanResponse,
-    summary="Scan a URL for phishing / malware",
-    tags=["Scanning"],
-)
-async def scan_url_full(request: URLScanRequestFull):
-    return await scan_url(request)
-
-
 # ─── POST /scan-sms ───────────────────────────────────────────────────────────
 
 @router.post(
@@ -229,10 +209,10 @@ async def scan_qr(request: QRScanRequest):
     t0 = time.monotonic()
 
     # Reuse URL scan logic
-    url_request = URLScanRequestFull(url=request.decoded_url, features={})
+    url_request = URLScanRequest(url=request.decoded_url, features={})
 
     try:
-        url_scan_result = await scan_url_full(url_request)
+        url_scan_result = await scan_url(url_request)
     except HTTPException:
         raise
     except Exception as e:

ml_models/sms_model.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ed863367581f7acb795ea5fd90283d37149949d6f3e38b3c18b410dfd737254
+oid sha256:6a61c57c417f8d3aa2feb613609202b688e689d6610fde57753a97863b512304
 size 231057

ml_models/train_url_classifier.py

@@ -103,10 +103,13 @@ def train_models(X, y, feature_cols):
     best_acc = max(rf_acc, xgb_acc)
     print(f"\n[+] Best model: {best_name} ({best_acc*100:.2f}%)")
 
-    joblib.dump(rf, os.path.join(MODEL_DIR, "url_rf_model.pkl"))
-    joblib.dump(xgb, os.path.join(MODEL_DIR, "url_xgb_model.pkl"))
-    joblib.dump(best, os.path.join(MODEL_DIR, "url_best_model.pkl"))
-    joblib.dump(feature_cols, os.path.join(MODEL_DIR, "url_feature_cols.pkl"))
+    os.makedirs(MODEL_DIR, exist_ok=True)
+    
+    # Use joblib consistently for all models to ensure compatibility
+    joblib.dump(rf, os.path.join(MODEL_DIR, "url_rf_model.pkl"), compress=3)
+    joblib.dump(xgb, os.path.join(MODEL_DIR, "url_xgb_model.pkl"), compress=3)
+    joblib.dump(best, os.path.join(MODEL_DIR, "url_best_model.pkl"), compress=3)
+    joblib.dump(feature_cols, os.path.join(MODEL_DIR, "url_feature_cols.pkl"), compress=3)
 
     with open(os.path.join(MODEL_DIR, "url_model_info.txt"), "w") as f:
         f.write(f"Best model: {best_name}\n")
@@ -114,9 +117,14 @@ def train_models(X, y, feature_cols):
         f.write(f"RF Accuracy: {rf_acc*100:.2f}%\n")
         f.write(f"XGB Accuracy: {xgb_acc*100:.2f}%\n")
         f.write(f"Features: {feature_cols}\n")
+        f.write(f"Model type saved: {type(best).__name__}\n")
 
     print("[+] Saved: url_best_model.pkl, url_rf_model.pkl, url_xgb_model.pkl, url_feature_cols.pkl")
 
+    # Verify that best is a proper sklearn model with required methods
+    if not hasattr(best, 'predict') or not hasattr(best, 'predict_proba'):
+        raise ValueError(f"ERROR: Saved model does not have required methods. Type: {type(best)}")
+    
     importances = pd.Series(best.feature_importances_, index=feature_cols)
     top5 = importances.nlargest(5)
     print("\n[*] Top 5 most important features:")

ml_models/url_best_model.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67299c44b7830768da2f21a6a4b934185f21e20206bbe7ad64f1b8397e32b127
-size 8387401
+oid sha256:e9e4a21643333d824248c3f892eb449ce4574e8ce0f9ec5785ea6f06afbaccd4
+size 1745423

ml_models/url_feature_cols.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a70fbbd450056aa56b42ccb03f9df051473fdc915ff954b71af168efd351dd7
-size 458
+oid sha256:54fa299350458e68d6bf45da4ae9a761159896b1c1cb78698ddc5c79bb8bf129
+size 332

ml_models/url_model_info.txt

@@ -3,3 +3,4 @@ Accuracy: 97.11%
 RF Accuracy: 97.11%
 XGB Accuracy: 96.83%
 Features: ['UsingIP', 'LongURL', 'ShortURL', 'Symbol@', 'Redirecting//', 'PrefixSuffix-', 'SubDomains', 'HTTPS', 'DomainRegLen', 'Favicon', 'NonStdPort', 'HTTPSDomainURL', 'RequestURL', 'AnchorURL', 'LinksInScriptTags', 'ServerFormHandler', 'InfoEmail', 'AbnormalURL', 'WebsiteForwarding', 'StatusBarCust', 'DisableRightClick', 'UsingPopupWindow', 'IframeRedirection', 'AgeofDomain', 'DNSRecording', 'WebsiteTraffic', 'PageRank', 'GoogleIndex', 'LinksPointingToPage', 'StatsReport']
+Model type saved: RandomForestClassifier

ml_models/url_rf_model.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67299c44b7830768da2f21a6a4b934185f21e20206bbe7ad64f1b8397e32b127
-size 8387401
+oid sha256:e9e4a21643333d824248c3f892eb449ce4574e8ce0f9ec5785ea6f06afbaccd4
+size 1745423

ml_models/url_xgb_model.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:863f307294721af7431889d8a2989581812a2149e9eb392762f37e35ab12b0d6
-size 255997
+oid sha256:2520fe1afc7aac37af02c6b138be94d2f137ccbfed0c114ab11e5d936eec5e96
+size 79523