Spaces:
Runtime error
Runtime error
Upload 10 files
Browse files- Version_5/app.py +90 -0
- Version_5/src/__pycache__/dom_scraper.cpython-313.pyc +0 -0
- Version_5/src/__pycache__/main.cpython-313.pyc +0 -0
- Version_5/src/__pycache__/orchestrator.cpython-313.pyc +0 -0
- Version_5/src/__pycache__/schemas.cpython-313.pyc +0 -0
- Version_5/src/__pycache__/sub_agents.cpython-313.pyc +0 -0
- Version_5/src/dom_scraper.py +28 -0
- Version_5/src/orchestrator.py +38 -0
- Version_5/src/schemas.py +12 -0
- Version_5/src/sub_agents.py +52 -0
Version_5/app.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
| 2 |
+
from fastapi import APIRouter, HTTPException
|
| 3 |
+
from pydantic import BaseModel, Field
|
| 4 |
+
|
| 5 |
+
# Absolute path imports mapped to your Mega-Server setup
|
| 6 |
+
from Version_5.src.dom_scraper import extract_dom_features
|
| 7 |
+
from Version_5.src.sub_agents import (
|
| 8 |
+
agent_url_analyst,
|
| 9 |
+
agent_html_structure,
|
| 10 |
+
agent_content_semantics,
|
| 11 |
+
agent_brand_impersonation
|
| 12 |
+
)
|
| 13 |
+
from Version_5.src.orchestrator import evaluate_consensus, run_judge
|
| 14 |
+
|
| 15 |
+
router = APIRouter()
|
| 16 |
+
|
| 17 |
+
# Define API request payload structure
|
| 18 |
+
class ThreatAnalysisRequest(BaseModel):
|
| 19 |
+
url: str = Field(..., description="Target landing page URL to analyze", example="http://example.com")
|
| 20 |
+
sender: str = Field(..., description="Alleged sender address header", example="security@paypal.com")
|
| 21 |
+
email_body: str = Field(..., description="Full text/body payload of the incoming message")
|
| 22 |
+
|
| 23 |
+
@router.post("/predict")
|
| 24 |
+
async def analyze_payload_endpoint(payload: ThreatAnalysisRequest):
|
| 25 |
+
url = payload.url.strip()
|
| 26 |
+
sender = payload.sender.strip()
|
| 27 |
+
email_body = payload.email_body.strip()
|
| 28 |
+
|
| 29 |
+
if not url and not email_body:
|
| 30 |
+
raise HTTPException(
|
| 31 |
+
status_code=400,
|
| 32 |
+
detail="Structural requirement breach: Provide at least a validation URL or a message body."
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
try:
|
| 36 |
+
start_time = time.perf_counter()
|
| 37 |
+
|
| 38 |
+
# Tier 1: Headless Chromium Capture
|
| 39 |
+
dom_data = extract_dom_features(url)
|
| 40 |
+
|
| 41 |
+
# Tier 2: Multi-Specialist Forensic Panel
|
| 42 |
+
reports = {
|
| 43 |
+
"URL_Agent": agent_url_analyst(url),
|
| 44 |
+
"HTML_Agent": agent_html_structure(dom_data),
|
| 45 |
+
"Content_Agent": agent_content_semantics(email_body),
|
| 46 |
+
"Brand_Agent": agent_brand_impersonation(email_body, sender)
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
# Tier 3: Core Consensus Evaluator
|
| 50 |
+
consensus_victory = evaluate_consensus(reports)
|
| 51 |
+
|
| 52 |
+
if consensus_victory:
|
| 53 |
+
# Handle standard serialization if agents return Pydantic objects or plain strings
|
| 54 |
+
final_verdict = {
|
| 55 |
+
"verdict": reports["URL_Agent"].claim if hasattr(reports["URL_Agent"], "claim") else str(reports["URL_Agent"]),
|
| 56 |
+
"confidence_score": reports["URL_Agent"].confidence if hasattr(reports["URL_Agent"], "confidence") else 1.0,
|
| 57 |
+
"justification": "Bypassed judicial review due to absolute sub-agent unanimity across forensics."
|
| 58 |
+
}
|
| 59 |
+
else:
|
| 60 |
+
# Tier 4: Judicial Override via Groq Cloud API
|
| 61 |
+
reports_str = "\n".join([
|
| 62 |
+
f"[{name}]\n{r.model_dump_json(indent=2) if hasattr(r, 'model_dump_json') else str(r)}"
|
| 63 |
+
for name, r in reports.items()
|
| 64 |
+
])
|
| 65 |
+
raw_data = f"Target URL: {url}\nTarget Sender: {sender}\nBody: {email_body}"
|
| 66 |
+
|
| 67 |
+
judge_verdict = run_judge(reports_str, raw_data)
|
| 68 |
+
final_verdict = judge_verdict.model_dump() if hasattr(judge_verdict, "model_dump") else judge_verdict
|
| 69 |
+
|
| 70 |
+
latency_ms = (time.perf_counter() - start_time) * 1000
|
| 71 |
+
|
| 72 |
+
# Safe response serialization
|
| 73 |
+
serializable_reports = {}
|
| 74 |
+
for name, report in reports.items():
|
| 75 |
+
serializable_reports[name] = report.model_dump() if hasattr(report, "model_dump") else str(report)
|
| 76 |
+
|
| 77 |
+
return {
|
| 78 |
+
"target_url": url,
|
| 79 |
+
"target_sender": sender,
|
| 80 |
+
"consensus_reached": consensus_victory,
|
| 81 |
+
"latency_ms": round(latency_ms, 2),
|
| 82 |
+
"sub_agent_claims": serializable_reports,
|
| 83 |
+
"final_evaluation": final_verdict
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
except Exception as e:
|
| 87 |
+
raise HTTPException(
|
| 88 |
+
status_code=500,
|
| 89 |
+
detail=f"Internal agent execution lifecycle crash: {str(e)}"
|
| 90 |
+
)
|
Version_5/src/__pycache__/dom_scraper.cpython-313.pyc
ADDED
|
Binary file (1.7 kB). View file
|
|
|
Version_5/src/__pycache__/main.cpython-313.pyc
ADDED
|
Binary file (4.8 kB). View file
|
|
|
Version_5/src/__pycache__/orchestrator.cpython-313.pyc
ADDED
|
Binary file (2.7 kB). View file
|
|
|
Version_5/src/__pycache__/schemas.cpython-313.pyc
ADDED
|
Binary file (1.37 kB). View file
|
|
|
Version_5/src/__pycache__/sub_agents.cpython-313.pyc
ADDED
|
Binary file (3.2 kB). View file
|
|
|
Version_5/src/dom_scraper.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from playwright.sync_api import sync_playwright
|
| 2 |
+
import json
|
| 3 |
+
|
| 4 |
+
def extract_dom_features(url: str) -> str:
|
| 5 |
+
if not url.startswith("http"):
|
| 6 |
+
return json.dumps({"error": "No valid URL provided."})
|
| 7 |
+
|
| 8 |
+
try:
|
| 9 |
+
with sync_playwright() as p:
|
| 10 |
+
browser = p.chromium.launch(headless=True)
|
| 11 |
+
page = browser.new_page()
|
| 12 |
+
# 10-second timeout to prevent stalling on dead phishing links
|
| 13 |
+
page.goto(url, timeout=10000)
|
| 14 |
+
|
| 15 |
+
# Extract where login forms actually send data
|
| 16 |
+
forms = page.locator("form").evaluate_all(
|
| 17 |
+
"elements => elements.map(e => ({ action: e.action, method: e.method }))"
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
# Extract invisible tracking/token fields
|
| 21 |
+
hidden_inputs = page.locator("input[type='hidden']").evaluate_all(
|
| 22 |
+
"elements => elements.map(e => ({ name: e.name, value: e.value }))"
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
browser.close()
|
| 26 |
+
return json.dumps({"forms": forms, "hidden_inputs": hidden_inputs})
|
| 27 |
+
except Exception as e:
|
| 28 |
+
return json.dumps({"scraper_error": str(e)})
|
Version_5/src/orchestrator.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from groq import Groq
|
| 3 |
+
from typing import Dict
|
| 4 |
+
from src.schemas import AgentReport, JudgeVerdict
|
| 5 |
+
|
| 6 |
+
client = Groq(api_key=os.environ.get("GROQ_API_KEY"))
|
| 7 |
+
|
| 8 |
+
def evaluate_consensus(reports: Dict[str, AgentReport]) -> bool:
|
| 9 |
+
claims = [r.claim for r in reports.values()]
|
| 10 |
+
confidences = [r.confidence for r in reports.values()]
|
| 11 |
+
|
| 12 |
+
all_same = len(set(claims)) == 1
|
| 13 |
+
high_conf = all(c >= 0.85 for c in confidences)
|
| 14 |
+
|
| 15 |
+
return all_same and high_conf
|
| 16 |
+
|
| 17 |
+
def run_judge(reports_summary: str, raw_data: str) -> JudgeVerdict:
|
| 18 |
+
system_prompt = (
|
| 19 |
+
"You are the presiding Judge Magistrate in an Enterprise Cybersecurity Matrix.\n"
|
| 20 |
+
"Your subordinate agents have provided split panel reports.\n"
|
| 21 |
+
"Respond strictly in valid JSON matching this schema:\n"
|
| 22 |
+
"{\"final_verdict\": \"CRITICAL_THREAT\"|\"SAFE\"|\"MANUAL_REVIEW_REQUIRED\", \"risk_score\": float, \"verdict_justification\": string}\n\n"
|
| 23 |
+
"DECISION ARCHITECTURE RULES:\n"
|
| 24 |
+
"1. If ANY analyst engine raises a 'PHISHING' claim with confidence >= 0.85, force 'CRITICAL_THREAT'.\n"
|
| 25 |
+
"2. Avoid label contradictions: Never set final_verdict to SAFE if your textual argument implies a threat pattern."
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
# We leverage the powerful 70B cloud model for premium reasoning accuracy
|
| 29 |
+
response = client.chat.completions.create(
|
| 30 |
+
model="llama-3.3-70b-versatile",
|
| 31 |
+
messages=[
|
| 32 |
+
{"role": "system", "content": system_prompt},
|
| 33 |
+
{"role": "user", "content": f"RAW INPUTS:\n{raw_data}\n\nREPORTS:\n{reports_summary}"}
|
| 34 |
+
],
|
| 35 |
+
response_format={"type": "json_object"},
|
| 36 |
+
temperature=0.0
|
| 37 |
+
)
|
| 38 |
+
return JudgeVerdict.model_validate_json(response.choices[0].message.content)
|
Version_5/src/schemas.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel, Field
|
| 2 |
+
from typing import List, Literal
|
| 3 |
+
|
| 4 |
+
class AgentReport(BaseModel):
|
| 5 |
+
claim: Literal["PHISHING", "LEGITIMATE", "AMBIGUOUS"]
|
| 6 |
+
confidence: float = Field(ge=0.0, le=1.0, description="Confidence score from 0.0 to 1.0")
|
| 7 |
+
evidence: List[str] = Field(description="Bullet points of forensic evidence found")
|
| 8 |
+
|
| 9 |
+
class JudgeVerdict(BaseModel):
|
| 10 |
+
final_verdict: Literal["CRITICAL_THREAT", "SAFE", "MANUAL_REVIEW_REQUIRED"]
|
| 11 |
+
risk_score: float = Field(ge=0.0, le=1.0)
|
| 12 |
+
verdict_justification: str = Field(description="Detailed explanation of the final decision")
|
Version_5/src/sub_agents.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from groq import Groq
|
| 3 |
+
from src.schemas import AgentReport
|
| 4 |
+
|
| 5 |
+
# Initialize the Groq Cloud Client
|
| 6 |
+
client = Groq(api_key=os.environ.get("GROQ_API_KEY"))
|
| 7 |
+
|
| 8 |
+
def run_agent(role_prompt: str, target_data: str) -> AgentReport:
|
| 9 |
+
# We call Groq's hosted 8B infrastructure instead of local resources
|
| 10 |
+
response = client.chat.completions.create(
|
| 11 |
+
model="llama-3.1-8b-instant",
|
| 12 |
+
messages=[
|
| 13 |
+
{"role": "system", "content": role_prompt},
|
| 14 |
+
{"role": "user", "content": target_data}
|
| 15 |
+
],
|
| 16 |
+
# Groq enforces JSON mode via this parameter block
|
| 17 |
+
response_format={"type": "json_object"},
|
| 18 |
+
temperature=0.0
|
| 19 |
+
)
|
| 20 |
+
return AgentReport.model_validate_json(response.choices[0].message.content)
|
| 21 |
+
|
| 22 |
+
def agent_url_analyst(url: str) -> AgentReport:
|
| 23 |
+
prompt = (
|
| 24 |
+
"You are a URL Shadows Analyst. Respond strictly in valid JSON matching this schema:\n"
|
| 25 |
+
"{\"claim\": \"PHISHING\"|\"LEGITIMATE\"|\"AMBIGUOUS\", \"confidence\": float, \"evidence\": [string]}\n"
|
| 26 |
+
"Analyze the structure for structural entropy, unusual domains, and brand keywords."
|
| 27 |
+
)
|
| 28 |
+
return run_agent(prompt, f"URL to analyze: {url}")
|
| 29 |
+
|
| 30 |
+
def agent_html_structure(dom_json: str) -> AgentReport:
|
| 31 |
+
prompt = (
|
| 32 |
+
"You are an HTML Code Analyst. Respond strictly in valid JSON matching this schema:\n"
|
| 33 |
+
"{\"claim\": \"PHISHING\"|\"LEGITIMATE\"|\"AMBIGUOUS\", \"confidence\": float, \"evidence\": [string]}\n"
|
| 34 |
+
"Analyze these DOM elements. Flag external tracking inputs or forms posting to alternative servers."
|
| 35 |
+
)
|
| 36 |
+
return run_agent(prompt, f"DOM Data: {dom_json}")
|
| 37 |
+
|
| 38 |
+
def agent_content_semantics(email_body: str) -> AgentReport:
|
| 39 |
+
prompt = (
|
| 40 |
+
"You are a Phishing Copywriter Critic. Respond strictly in valid JSON matching this schema:\n"
|
| 41 |
+
"{\"claim\": \"PHISHING\"|\"LEGITIMATE\"|\"AMBIGUOUS\", \"confidence\": float, \"evidence\": [string]}\n"
|
| 42 |
+
"Extract semantic compliance anomalies, high emotional coercion markers, and urgency loops."
|
| 43 |
+
)
|
| 44 |
+
return run_agent(prompt, f"Email Body: {email_body}")
|
| 45 |
+
|
| 46 |
+
def agent_brand_impersonation(email_body: str, sender: str) -> AgentReport:
|
| 47 |
+
prompt = (
|
| 48 |
+
"You are an Identity Protection Agent. Respond strictly in valid JSON matching this schema:\n"
|
| 49 |
+
"{\"claim\": \"PHISHING\"|\"LEGITIMATE\"|\"AMBIGUOUS\", \"confidence\": float, \"evidence\": [string]}\n"
|
| 50 |
+
"Flag mismatched target namespaces where sender domains do not match corporate identifiers."
|
| 51 |
+
)
|
| 52 |
+
return run_agent(prompt, f"Sender: {sender}\nBody: {email_body}")
|