OR implementation 1

app.py

@@ -12,6 +12,9 @@ import sqlite3
 import re
 from langdetect import detect, DetectorFactory
 from urllib.parse import urlparse, urlunparse
+import time
+from openrouter import OpenRouter
+import requests
 
 # Make langdetect deterministic
 DetectorFactory.seed = 0
@@ -32,6 +35,16 @@ model = None
 translator_tokenizer = None
 translator_model = None
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# OpenRouter client + model ids (env-configurable placeholders)
+or_client = None
+OR_MODEL_1 = os.environ.get("OR_MODEL_1", "or-model-1")
+OR_MODEL_2 = os.environ.get("OR_MODEL_2", "or-model-2")
+OR_TIMEOUT = int(os.environ.get("OR_TIMEOUT", "15"))
+OR_CONSECUTIVE_FAILURES = 0
+OR_CB_THRESHOLD = int(os.environ.get("OR_CB_THRESHOLD", "5"))
+OR_CB_OPEN_SECONDS = int(os.environ.get("OR_CB_OPEN_SECONDS", "300"))
+OR_LAST_ERROR = None
+OR_CB_OPEN_UNTIL = 0
 
 # -----------------------
 # DB + URL helpers
@@ -355,6 +368,117 @@ def predict_phishing(text: str):
         logger.error(f"Error predicting phishing: {e}")
         return "Error", 0.0
 
+
+def generate_text(prompt: str, model_id: str, timeout: int = OR_TIMEOUT):
+    """Send prompt to OpenRouter HTTP API and return dict {ok, text, error, status}.
+    Uses chat completions endpoint and returns the assistant message content.
+    Falls back gracefully on network or parsing errors.
+    """
+    global OR_CONSECUTIVE_FAILURES, OR_LAST_ERROR, OR_CB_OPEN_UNTIL
+    api_key = os.environ.get("OPENROUTER_API_KEY")
+    # circuit breaker: short-circuit if open
+    if OR_CB_OPEN_UNTIL and time.time() < OR_CB_OPEN_UNTIL:
+        return {"ok": False, "text": None, "error": "circuit_open", "status": None}
+    if not api_key:
+        OR_CONSECUTIVE_FAILURES += 1
+        OR_LAST_ERROR = "Missing OPENROUTER_API_KEY"
+        return {"ok": False, "text": None, "error": "Missing OPENROUTER_API_KEY", "status": None}
+    url = "https://api.openrouter.ai/v1/chat/completions"
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+    body = {
+        "model": model_id,
+        "messages": [
+            {"role": "system", "content": "You are an automated phishing detector. Respond with only valid JSON."},
+            {"role": "user", "content": prompt}
+        ],
+        "temperature": 0.0,
+        "max_tokens": 256
+    }
+    try:
+        resp = requests.post(url, headers=headers, json=body, timeout=timeout)
+        text = resp.text
+        if resp.status_code != 200:
+            logger.error(f"OpenRouter API error {resp.status_code}: {text}")
+            OR_CONSECUTIVE_FAILURES += 1
+            OR_LAST_ERROR = f"status {resp.status_code}"
+            if OR_CONSECUTIVE_FAILURES >= OR_CB_THRESHOLD:
+                OR_CB_OPEN_UNTIL = time.time() + OR_CB_OPEN_SECONDS
+                logger.warning("OpenRouter circuit opened due to repeated failures")
+            return {"ok": False, "text": text, "error": f"status {resp.status_code}", "status": resp.status_code}
+        # try to extract assistant content
+        j = resp.json()
+        content = None
+        if isinstance(j, dict) and "choices" in j and isinstance(j["choices"], list) and j["choices"]:
+            first = j["choices"][0]
+            # chat-style: message.content
+            if "message" in first and isinstance(first["message"], dict):
+                content = first["message"].get("content")
+            # or text field
+            if not content:
+                content = first.get("text")
+        # success -> reset failure counter
+        OR_CONSECUTIVE_FAILURES = 0
+        OR_LAST_ERROR = None
+        return {"ok": True, "text": content or text, "error": None, "status": resp.status_code}
+    except Exception as e:
+        logger.error(f"OpenRouter request error: {e}")
+        OR_CONSECUTIVE_FAILURES += 1
+        OR_LAST_ERROR = str(e)
+        if OR_CONSECUTIVE_FAILURES >= OR_CB_THRESHOLD:
+            OR_CB_OPEN_UNTIL = time.time() + OR_CB_OPEN_SECONDS
+            logger.warning("OpenRouter circuit opened due to repeated exceptions")
+        return {"ok": False, "text": None, "error": str(e), "status": None}
+
+
+def predict_phishing_or(text: str, model_id: str):
+    """Call OpenRouter to classify text. Return (label, confidence_percent, raw_response).
+    Falls back to HF `predict_phishing` on error or parse failure.
+    """
+    # craft a strict prompt that asks for JSON
+    prompt = (
+        "Classify the following text as either 'Phishing' or 'Safe'.\n"
+        "Return ONLY a single JSON object with exactly these fields: {\"label\": \"Phishing\"|\"Safe\", \"confidence\": <0-100 float>}\n"
+        "Do not add any other text or explanation.\n\n"
+        f"TEXT:\n{text}"
+    )
+    resp = generate_text(prompt, model_id)
+    logger.info(f"OpenRouter raw response: {resp}")
+    if not resp.get("ok") or not resp.get("text"):
+        # fallback to HF
+        label, conf = predict_phishing(text)
+        return label, conf, {"fallback": True, "reason": resp.get("error"), "raw": resp.get("text")}
+
+    # try to parse JSON object from resp['text']
+    txt = resp.get("text")
+    # extract the first {...} block
+    import json, re
+    m = re.search(r"\{.*\}", txt, re.DOTALL)
+    if not m:
+        label, conf = predict_phishing(text)
+        return label, conf, {"fallback": True, "reason": "no_json", "raw": txt}
+    try:
+        obj = json.loads(m.group(0))
+        lab = obj.get("label")
+        conf = obj.get("confidence")
+        if isinstance(conf, str):
+            try:
+                conf = float(conf)
+            except Exception:
+                conf = 0.0
+        if lab and (lab.lower() in ("phishing", "safe")):
+            label = "Phishing" if lab.lower() == "phishing" else "Safe"
+            conf_val = round(float(conf), 1) if conf is not None else 0.0
+            return label, conf_val, {"fallback": False, "raw": txt}
+        else:
+            raise ValueError("Invalid label")
+    except Exception as e:
+        logger.error(f"Error parsing OpenRouter JSON: {e} -- raw: {txt}")
+        label, conf = predict_phishing(text)
+        return label, conf, {"fallback": True, "reason": str(e), "raw": txt}
+
 # -----------------------
 # Unified pipeline
 # -----------------------
@@ -487,7 +611,18 @@ def home():
 
 @app.route("/health", methods=["GET"])
 def health():
-    return jsonify({"status": "healthy", "model_loaded": model is not None})
+    or_configured = bool(os.environ.get("OPENROUTER_API_KEY"))
+    return jsonify({
+        "status": "healthy",
+        "model_loaded": model is not None,
+        "openrouter": {
+            "configured": or_configured,
+            "client_initialized": or_client is not None,
+            "consecutive_failures": OR_CONSECUTIVE_FAILURES,
+            "circuit_open": bool(OR_CB_OPEN_UNTIL and time.time() < OR_CB_OPEN_UNTIL),
+            "last_error": OR_LAST_ERROR
+        }
+    })
 
 
 
@@ -508,6 +643,158 @@ def analyze():
         return jsonify({"error": "Internal server error"}), 500
 
 
+def analyze_pipeline_with_model(message: str, model_id: str):
+    """Same pipeline as analyze_pipeline but uses the specified OpenRouter model for classification.
+    Falls back to HF classifier when OR fails.
+    """
+    try:
+        text = (message or "").strip()
+        if not text:
+            return {"error": "empty", "blacklist": False}
+
+        # extract both full URLs and domains once
+        full_urls, domains = extract_urls_and_domains(text)
+
+        # --- load whitelist set (lowercased) ---
+        try:
+            conn = get_db_connection()
+            cursor = conn.cursor()
+            cursor.execute("SELECT LOWER(domain) FROM whitelist")
+            rows = cursor.fetchall()
+            whitelist_set = {r[0] for r in rows if r and r[0]}
+            conn.close()
+        except Exception as e:
+            logger.warning(f"Whitelist load failed: {e}")
+            whitelist_set = set()
+
+        # 1) domain whitelist check (allow subdomains)
+        if domains and whitelist_set:
+            for domain in domains:
+                d = (domain or "").lower().strip().strip('/')
+                if not d:
+                    continue
+                # direct or subdomain match
+                matched_whitelist = False
+                if d in whitelist_set:
+                    matched_whitelist = True
+                else:
+                    for wl in whitelist_set:
+                        if d == wl or d.endswith("." + wl):
+                            matched_whitelist = True
+                            break
+
+                if matched_whitelist:
+                    # BEFORE returning Safe, check whether any full URL in the message
+                    # is exactly present in the blacklist (normalized). If so, treat as Phishing.
+                    if full_urls:
+                        try:
+                            bconn = get_blacklist_connection()
+                            bcur = bconn.cursor()
+                            for u in full_urls:
+                                norm_u = normalize_full_url(u).lower()
+                                bcur.execute("SELECT 1 FROM blacklist WHERE LOWER(url) = ?", (norm_u,))
+                                if bcur.fetchone():
+                                    bconn.close()
+                                    return {
+                                        "result": "Phishing",
+                                        "confidence": "100.0%",
+                                        "message": text,
+                                        "blacklist": True,
+                                        "whitelist": False,
+                                        "detected_lang": None,
+                                        "translated_text": None,
+                                        "model_used": model_id
+                                    }
+                            bconn.close()
+                        except Exception as e:
+                            logger.warning(f"Blacklist-full-url check failed: {e}")
+                    # If no full-url blacklist hit, return Safe because domain is whitelisted
+                    return {
+                        "result": "Safe",
+                        "confidence": "100.0%",
+                        "message": text,
+                        "blacklist": False,
+                        "whitelist": True,
+                        "detected_lang": None,
+                        "translated_text": None,
+                        "model_used": model_id
+                    }
+
+        # 2) not whitelisted by domain (or whitelist didn't apply) -> regular blacklist check
+        is_black, _reason = is_blacklisted(text)
+        if is_black:
+            return {
+                "result": "Phishing",
+                "confidence": "100.0%",
+                "message": text,
+                "blacklist": True,
+                "whitelist": False,
+                "detected_lang": None,
+                "translated_text": None,
+                "model_used": model_id
+            }
+
+        # 3) language detection + optional translate
+        lang = detect_language(text)
+        translated_text = None
+        analysis_text = text
+        if lang in ("tl", "fil"):
+            translated_text = translate_tl_to_en(text)
+            analysis_text = translated_text or text
+
+        # 4) classification via OpenRouter model (with HF fallback inside)
+        label, conf, raw = predict_phishing_or(analysis_text, model_id)
+        return {
+            "result": label,
+            "confidence": f"{conf}%",
+            "message": text,
+            "blacklist": False,
+            "whitelist": False,
+            "detected_lang": lang,
+            "translated_text": translated_text,
+            "model_used": model_id,
+            "or_raw": raw
+        }
+
+    except Exception as e:
+        logger.error(f"Pipeline error: {e}")
+        return {"result": "Error", "confidence": "0%", "message": message, "blacklist": False, "whitelist": False}
+
+
+@app.route("/analyze/or1", methods=["POST"])
+def analyze_or1():
+    try:
+        data = request.get_json()
+        if not data or "message" not in data:
+            return jsonify({"error": "Missing 'message' field"}), 400
+        message = data["message"]
+        if not message or not message.strip():
+            return jsonify({"error": "Message cannot be empty"}), 400
+
+        result = analyze_pipeline_with_model(message, OR_MODEL_1)
+        return jsonify(result), 200
+    except Exception as e:
+        logger.error(f"Error in analyze/or1 endpoint: {e}")
+        return jsonify({"error": "Internal server error"}), 500
+
+
+@app.route("/analyze/or2", methods=["POST"])
+def analyze_or2():
+    try:
+        data = request.get_json()
+        if not data or "message" not in data:
+            return jsonify({"error": "Missing 'message' field"}), 400
+        message = data["message"]
+        if not message or not message.strip():
+            return jsonify({"error": "Message cannot be empty"}), 400
+
+        result = analyze_pipeline_with_model(message, OR_MODEL_2)
+        return jsonify(result), 200
+    except Exception as e:
+        logger.error(f"Error in analyze/or2 endpoint: {e}")
+        return jsonify({"error": "Internal server error"}), 500
+
+
 # =============================
 # /evaluate (GET form + POST CSV)
 # =============================
@@ -698,4 +985,16 @@ def internal_error(error):
 # -----------------------
 # Startup
 # -----------------------
-load_model()
+load_model()
+try:
+    api_key = os.environ.get("OPENROUTER_API_KEY")
+    if api_key:
+        try:
+            or_client = OpenRouter(api_key=api_key)
+            logger.info("OpenRouter client initialized.")
+        except Exception as e:
+            logger.warning(f"Failed to initialize OpenRouter client object: {e}")
+    else:
+        logger.warning("OPENROUTER_API_KEY not set; OpenRouter endpoints will still attempt HTTP calls but client object is not created.")
+except Exception as e:
+    logger.warning(f"OpenRouter init warning: {e}")

requirements.txt

@@ -10,4 +10,6 @@ google-auth-oauthlib
 google-auth-httplib2
 langdetect
 sentencepiece
-sacremoses
+sacremoses
+openrouter
+requests

test_api.py

@@ -0,0 +1,28 @@
+import os
+import requests
+import sys
+import json
+
+API_BASE = os.getenv("API_BASE", "http://localhost:8080")
+MESSAGE = os.getenv("TEST_MESSAGE", "Please verify your account at http://example.com/login")
+TIMEOUT = int(os.getenv("TEST_TIMEOUT", "10"))
+
+endpoints = ["/analyze", "/analyze/or1", "/analyze/or2"]
+
+print(f"Testing API at {API_BASE} with message: {MESSAGE}\n")
+for ep in endpoints:
+    url = API_BASE.rstrip("/") + ep
+    payload = {"message": MESSAGE}
+    try:
+        resp = requests.post(url, json=payload, timeout=TIMEOUT)
+        try:
+            body = resp.json()
+        except Exception:
+            body = resp.text
+        print(f"Endpoint: {ep} -> Status: {resp.status_code}")
+        print(json.dumps(body, indent=2, ensure_ascii=False))
+    except requests.exceptions.RequestException as e:
+        print(f"Endpoint: {ep} -> Request failed: {e}")
+    print("---")
+
+print("Done.")