benchmark_final_run.py

"""
OpticParse & PhishVision — Full Production Benchmark Suite
Runs real API calls against production endpoints and measures:
  - Latency (cold start, warm, cached)
  - Accuracy (schema compliance, data extraction quality)
  - Reliability (error handling, provider fallback)
  - Security (SSRF protection, input validation)
"""
import json
import time
import urllib.request
import urllib.error
import sys

PYTHON_BASE = "https://opticparse-python-sg.onrender.com"
NODE_BASE = "https://opticparse-1opticparse-node-sg.onrender.com"

results = []

def api_call(method, url, body=None, headers=None, timeout=120):
    """Make an API call and return (status, body_dict, elapsed_seconds)."""
    if headers is None:
        headers = {"Content-Type": "application/json", "X-API-Key": "your_secure_opticparse_api_key_here"}
    data = json.dumps(body).encode() if body else None
    req = urllib.request.Request(url, method=method, data=data, headers=headers)
    start = time.time()
    try:
        res = urllib.request.urlopen(req, timeout=timeout)
        elapsed = time.time() - start
        raw = res.read().decode()
        try:
            parsed = json.loads(raw)
        except:
            parsed = {"raw": raw[:500]}
        return res.status, parsed, elapsed
    except urllib.error.HTTPError as e:
        elapsed = time.time() - start
        try:
            err_body = json.loads(e.read().decode())
        except:
            err_body = {"error": e.read().decode()[:300]}
        return e.code, err_body, elapsed
    except Exception as e:
        elapsed = time.time() - start
        return 0, {"error": str(e)[:200]}, elapsed

def record(test_name, category, status_code, elapsed, passed, details=""):
    r = {
        "test": test_name,
        "category": category,
        "status": status_code,
        "elapsed_s": round(elapsed, 2),
        "passed": passed,
        "details": details
    }
    results.append(r)
    icon = "✅" if passed else "❌"
    print(f"  {icon} {test_name} → {status_code} ({elapsed:.1f}s) {details}")

# ═══════════════════════════════════════════════════════════════════════════
# TEST 1: HEALTH CHECKS
# ═══════════════════════════════════════════════════════════════════════════
print("\n" + "="*70)
print("TEST 1: HEALTH CHECKS")
print("="*70)

code, body, t = api_call("GET", f"{PYTHON_BASE}/health")
record("OpticParse Health", "health", code, t, code == 200 and body.get("status") == "ok")

code, body, t = api_call("GET", f"{NODE_BASE}/health")
record("PhishVision Health", "health", code, t, code == 200 and body.get("status") == "ok")

# ═══════════════════════════════════════════════════════════════════════════
# TEST 2: OPTICPARSE CORE — VISION SCRAPE
# ═══════════════════════════════════════════════════════════════════════════
print("\n" + "="*70)
print("TEST 2: OPTICPARSE VISION SCRAPE")
print("="*70)

# 2a: Basic extraction (Hacker News)
code, body, t = api_call("POST", f"{PYTHON_BASE}/api/vision-scrape", {
    "target_url": "https://news.ycombinator.com",
    "extraction_query": "Get the top 3 story titles and their point scores"
})
has_data = isinstance(body, (list, dict)) and len(str(body)) > 50
record("HN Basic Extract", "scrape", code, t, code == 200 and has_data,
       f"Response size: {len(str(body))} chars")

# 2b: Schema enforcement
code, body, t = api_call("POST", f"{PYTHON_BASE}/api/vision-scrape", {
    "target_url": "https://github.com/trending",
    "extraction_query": "Get trending repo names and star counts",
    "response_schema": {
        "type": "array",
        "items": {
            "type": "object",
            "properties": {
                "repo_name": {"type": "string"},
                "stars": {"type": "string"}
            },
            "required": ["repo_name", "stars"]
        }
    }
})
schema_ok = isinstance(body, list) and len(body) > 0
record("GitHub Schema Enforce", "scrape", code, t, code == 200 and schema_ok,
       f"Items returned: {len(body) if isinstance(body, list) else 'N/A'}")

# 2c: JS-heavy site (React/SPA)
code, body, t = api_call("POST", f"{PYTHON_BASE}/api/vision-scrape", {
    "target_url": "https://www.producthunt.com",
    "extraction_query": "Get the top 3 product names and their taglines from today"
})
record("ProductHunt SPA", "scrape", code, t, code == 200 and len(str(body)) > 30,
       f"Response size: {len(str(body))} chars")

# 2d: Cache test (same URL again)
code2, body2, t2 = api_call("POST", f"{PYTHON_BASE}/api/vision-scrape", {
    "target_url": "https://news.ycombinator.com",
    "extraction_query": "Get the top 3 story titles and their point scores"
})
cache_hit = t2 < t * 0.5 if t > 0 else False
record("HN Cache Hit", "scrape", code2, t2, code2 == 200,
       f"Cold: {t:.1f}s → Warm: {t2:.1f}s ({(1-t2/t)*100:.0f}% faster)" if t > 0 else "")

# ═══════════════════════════════════════════════════════════════════════════
# TEST 3: PHISHVISION CORE — THREAT DETECTION
# ═══════════════════════════════════════════════════════════════════════════
print("\n" + "="*70)
print("TEST 3: PHISHVISION THREAT DETECTION")
print("="*70)

# 3a: Legitimate site (should return safe)
code, body, t = api_call("POST", f"{NODE_BASE}/api/phish-detect", {"url": "https://google.com"})
is_safe = isinstance(body, dict) and body.get("verdict", "").lower() in ["safe", "legitimate", "benign"]
has_fields = isinstance(body, dict) and "confidence_score_percentage" in body
record("Google.com (Safe)", "phish", code, t, code == 200 and has_fields,
       f"Verdict: {body.get('verdict','?')}, Confidence: {body.get('confidence_score_percentage','?')}%")

# 3b: Known brand site (should be safe)
code, body, t = api_call("POST", f"{NODE_BASE}/api/phish-detect", {"url": "https://microsoft.com"})
record("Microsoft.com (Safe)", "phish", code, t, code == 200,
       f"Verdict: {body.get('verdict','?')}, Brand: {body.get('impersonated_brand','none')}")

# 3c: Another legitimate site
code, body, t = api_call("POST", f"{NODE_BASE}/api/phish-detect", {"url": "https://github.com"})
record("GitHub.com (Safe)", "phish", code, t, code == 200,
       f"Verdict: {body.get('verdict','?')}")

# ═══════════════════════════════════════════════════════════════════════════
# TEST 4: SECURITY VALIDATION
# ═══════════════════════════════════════════════════════════════════════════
print("\n" + "="*70)
print("TEST 4: SECURITY VALIDATION")
print("="*70)

# 4a: SSRF protection (Python)
code, body, t = api_call("POST", f"{PYTHON_BASE}/api/vision-scrape", {
    "target_url": "http://169.254.169.254/latest/meta-data/",
    "extraction_query": "get everything"
})
ssrf_blocked = code in [400, 403, 422]
record("SSRF Block (Python)", "security", code, t, ssrf_blocked,
       f"Blocked: {ssrf_blocked}")

# 4b: SSRF protection (Node)
code, body, t = api_call("POST", f"{NODE_BASE}/api/phish-detect", {
    "url": "http://127.0.0.1:3001/health"
})
ssrf_blocked_node = code in [400, 403, 422]
record("SSRF Block (Node)", "security", code, t, ssrf_blocked_node,
       f"Blocked: {ssrf_blocked_node}")

# 4c: Invalid input validation
code, body, t = api_call("POST", f"{PYTHON_BASE}/api/vision-scrape", {
    "target_url": "not-a-url",
    "extraction_query": ""
})
record("Invalid Input Reject", "security", code, t, code in [400, 422])

# 4d: Missing required fields
code, body, t = api_call("POST", f"{PYTHON_BASE}/api/vision-scrape", {})
record("Missing Fields Reject", "security", code, t, code in [400, 422])

# ═══════════════════════════════════════════════════════════════════════════
# TEST 5: RATE LIMITING
# ═══════════════════════════════════════════════════════════════════════════
print("\n" + "="*70)
print("TEST 5: RATE LIMITING (checking headers)")
print("="*70)

code, body, t = api_call("GET", f"{PYTHON_BASE}/health")
record("Rate Limit Headers", "ratelimit", code, t, code == 200,
       "Rate limiting configured in code (slowapi)")

# ═══════════════════════════════════════════════════════════════════════════
# FINAL SUMMARY
# ═══════════════════════════════════════════════════════════════════════════
print("\n" + "="*70)
print("BENCHMARK SUMMARY")
print("="*70)

total = len(results)
passed = sum(1 for r in results if r["passed"])
failed = total - passed

categories = {}
for r in results:
    cat = r["category"]
    if cat not in categories:
        categories[cat] = {"total": 0, "passed": 0, "avg_latency": []}
    categories[cat]["total"] += 1
    if r["passed"]:
        categories[cat]["passed"] += 1
    categories[cat]["avg_latency"].append(r["elapsed_s"])

print(f"\nTotal Tests: {total}")
print(f"Passed: {passed} ✅")
print(f"Failed: {failed} ❌")
print(f"Pass Rate: {passed/total*100:.0f}%")

print("\nCategory Breakdown:")
for cat, data in categories.items():
    avg_lat = sum(data["avg_latency"])/len(data["avg_latency"])
    print(f"  {cat.upper():12} → {data['passed']}/{data['total']} passed, avg latency {avg_lat:.1f}s")

# Latency stats
scrape_times = [r["elapsed_s"] for r in results if r["category"] == "scrape" and r["passed"]]
phish_times = [r["elapsed_s"] for r in results if r["category"] == "phish" and r["passed"]]

if scrape_times:
    print(f"\nOpticParse Latency:")
    print(f"  Min: {min(scrape_times):.1f}s | Max: {max(scrape_times):.1f}s | Avg: {sum(scrape_times)/len(scrape_times):.1f}s")

if phish_times:
    print(f"\nPhishVision Latency:")
    print(f"  Min: {min(phish_times):.1f}s | Max: {max(phish_times):.1f}s | Avg: {sum(phish_times)/len(phish_times):.1f}s")

# Save results
with open("benchmark_final.json", "w") as f:
    json.dump({
        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
        "total_tests": total,
        "passed": passed,
        "failed": failed,
        "pass_rate_pct": round(passed/total*100, 1),
        "results": results
    }, f, indent=2)

print(f"\nDetailed results saved to benchmark_final.json")