scripts/smoke_real_ai.py

"""
Smoke test for DocDoe Real AI Mode (OpenRouter).

Checks that the backend is running with a real AI provider (OpenRouter),
not mock/demo mode. Hits each brain endpoint and validates:
1. /health/ai reports "ready" status
2. Each generation endpoint returns a real model name (not "mock")
3. is_fallback is False on all responses
4. Output quality: responses contain non-template content

Usage:
    python scripts/smoke_real_ai.py [--base-url http://localhost:8000]

Exit codes:
    0 = all checks pass (real AI mode confirmed)
    1 = at least one check failed (mock/fallback detected)
"""

from __future__ import annotations

import argparse
import json
import sys
from urllib.request import Request, urlopen
from urllib.error import URLError


def _request(base: str, method: str, path: str, body: dict | None = None) -> dict:
    url = f"{base}{path}"
    data = json.dumps(body).encode() if body else None
    headers = {"Content-Type": "application/json"} if data else {}
    req = Request(url, data=data, headers=headers, method=method)
    try:
        with urlopen(req, timeout=30) as resp:
            return json.loads(resp.read().decode())
    except URLError as exc:
        return {"_error": str(exc)}


def _get(base: str, path: str) -> dict:
    return _request(base, "GET", path)


def _post(base: str, path: str, body: dict) -> dict:
    return _request(base, "POST", path, body)


def _check(label: str, ok: bool, detail: str = "") -> bool:
    status = "PASS" if ok else "FAIL"
    suffix = f" — {detail}" if detail else ""
    print(f"  [{status}] {label}{suffix}")
    return ok


def _is_real_model(model: str) -> bool:
    return model.startswith("openrouter:")


def main() -> None:
    parser = argparse.ArgumentParser(description="DocDoe Real AI Mode Smoke Test")
    parser.add_argument("--base-url", default="http://localhost:8000")
    args = parser.parse_args()
    base = args.base_url.rstrip("/")

    print(f"\nDocDoe Real AI Smoke Test (OpenRouter)")
    print(f"Target: {base}\n")

    all_pass = True

    # 1. Health check
    print("1. AI Health")
    health = _get(base, "/health/ai")
    if "_error" in health:
        _check("Reachable", False, health["_error"])
        print("\nBackend is not running. Start it first.")
        sys.exit(1)

    ai_status = health.get("status", "unknown")
    all_pass &= _check("AI status", ai_status == "ready", f"status={ai_status}")
    all_pass &= _check("Provider", health.get("provider") == "openrouter", f"provider={health.get('provider')}")
    all_pass &= _check("Model configured", bool(health.get("model_main")), f"model={health.get('model_main')}")
    print()

    # 2. Ask endpoint
    print("2. Ask DocDoe")
    ask = _post(base, "/ask", {"question": "What is photosynthesis?", "mode": "explain_simple"})
    if "_error" not in ask:
        model = ask.get("model_used", "")
        fallback = ask.get("is_fallback", True)
        answer = ask.get("answer", "")
        all_pass &= _check("Model is real", _is_real_model(model), f"model_used={model}")
        all_pass &= _check("Not fallback", not fallback, f"is_fallback={fallback}")
        all_pass &= _check("Answer length", len(answer) > 50, f"len={len(answer)}")
    else:
        all_pass &= _check("Ask reachable", False, ask["_error"])
    print()

    # 3. Studio endpoints
    studio_tests = [
        ("Notes", "/generate/notes", {"topic": "Photosynthesis", "language_preference": "English"}),
        ("Quiz", "/generate/quiz", {"topic": "Photosynthesis", "language_preference": "English", "options": {"question_count": 3}}),
        ("Flashcards", "/generate/flashcards", {"topic": "Photosynthesis", "language_preference": "English", "options": {"card_count": 3}}),
        ("Exam Answer", "/generate/exam-answer", {"topic": "Photosynthesis", "language_preference": "English"}),
        ("Simple Explanation", "/generate/simple-explanation", {"topic": "Photosynthesis", "language_preference": "English"}),
        ("Last Night Plan", "/generate/last-night-plan", {"topic": "Photosynthesis", "time_left": "tonight", "language_preference": "English"}),
    ]

    for idx, (name, path, body) in enumerate(studio_tests, start=3):
        print(f"{idx}. Studio: {name}")
        resp = _post(base, path, body)
        if "_error" not in resp:
            model = resp.get("model_used", "")
            fallback = resp.get("is_fallback", True)
            output = resp.get("output", {})
            all_pass &= _check("Model is real", _is_real_model(model), f"model_used={model}")
            all_pass &= _check("Not fallback", not fallback, f"is_fallback={fallback}")
            all_pass &= _check("Has output", bool(output), f"keys={list(output.keys())[:5]}")
        else:
            all_pass &= _check(f"{name} reachable", False, resp["_error"])
        print()

    # Summary
    print("=" * 50)
    if all_pass:
        print("RESULT: ALL CHECKS PASSED — Real AI mode confirmed")
        print("DocDoe is generating real AI answers via OpenRouter.")
    else:
        print("RESULT: SOME CHECKS FAILED — Mock/fallback detected")
        print("Check AI_PROVIDER, OPENROUTER_API_KEY, and backend logs.")
    print()
    sys.exit(0 if all_pass else 1)


if __name__ == "__main__":
    main()