""" Smoke test for DocDoe Real AI Mode (OpenRouter). Checks that the backend is running with a real AI provider (OpenRouter), not mock/demo mode. Hits each brain endpoint and validates: 1. /health/ai reports "ready" status 2. Each generation endpoint returns a real model name (not "mock") 3. is_fallback is False on all responses 4. Output quality: responses contain non-template content Usage: python scripts/smoke_real_ai.py [--base-url http://localhost:8000] Exit codes: 0 = all checks pass (real AI mode confirmed) 1 = at least one check failed (mock/fallback detected) """ from __future__ import annotations import argparse import json import sys from urllib.request import Request, urlopen from urllib.error import URLError def _request(base: str, method: str, path: str, body: dict | None = None) -> dict: url = f"{base}{path}" data = json.dumps(body).encode() if body else None headers = {"Content-Type": "application/json"} if data else {} req = Request(url, data=data, headers=headers, method=method) try: with urlopen(req, timeout=30) as resp: return json.loads(resp.read().decode()) except URLError as exc: return {"_error": str(exc)} def _get(base: str, path: str) -> dict: return _request(base, "GET", path) def _post(base: str, path: str, body: dict) -> dict: return _request(base, "POST", path, body) def _check(label: str, ok: bool, detail: str = "") -> bool: status = "PASS" if ok else "FAIL" suffix = f" — {detail}" if detail else "" print(f" [{status}] {label}{suffix}") return ok def _is_real_model(model: str) -> bool: return model.startswith("openrouter:") def main() -> None: parser = argparse.ArgumentParser(description="DocDoe Real AI Mode Smoke Test") parser.add_argument("--base-url", default="http://localhost:8000") args = parser.parse_args() base = args.base_url.rstrip("/") print(f"\nDocDoe Real AI Smoke Test (OpenRouter)") print(f"Target: {base}\n") all_pass = True # 1. Health check print("1. AI Health") health = _get(base, "/health/ai") if "_error" in health: _check("Reachable", False, health["_error"]) print("\nBackend is not running. Start it first.") sys.exit(1) ai_status = health.get("status", "unknown") all_pass &= _check("AI status", ai_status == "ready", f"status={ai_status}") all_pass &= _check("Provider", health.get("provider") == "openrouter", f"provider={health.get('provider')}") all_pass &= _check("Model configured", bool(health.get("model_main")), f"model={health.get('model_main')}") print() # 2. Ask endpoint print("2. Ask DocDoe") ask = _post(base, "/ask", {"question": "What is photosynthesis?", "mode": "explain_simple"}) if "_error" not in ask: model = ask.get("model_used", "") fallback = ask.get("is_fallback", True) answer = ask.get("answer", "") all_pass &= _check("Model is real", _is_real_model(model), f"model_used={model}") all_pass &= _check("Not fallback", not fallback, f"is_fallback={fallback}") all_pass &= _check("Answer length", len(answer) > 50, f"len={len(answer)}") else: all_pass &= _check("Ask reachable", False, ask["_error"]) print() # 3. Studio endpoints studio_tests = [ ("Notes", "/generate/notes", {"topic": "Photosynthesis", "language_preference": "English"}), ("Quiz", "/generate/quiz", {"topic": "Photosynthesis", "language_preference": "English", "options": {"question_count": 3}}), ("Flashcards", "/generate/flashcards", {"topic": "Photosynthesis", "language_preference": "English", "options": {"card_count": 3}}), ("Exam Answer", "/generate/exam-answer", {"topic": "Photosynthesis", "language_preference": "English"}), ("Simple Explanation", "/generate/simple-explanation", {"topic": "Photosynthesis", "language_preference": "English"}), ("Last Night Plan", "/generate/last-night-plan", {"topic": "Photosynthesis", "time_left": "tonight", "language_preference": "English"}), ] for idx, (name, path, body) in enumerate(studio_tests, start=3): print(f"{idx}. Studio: {name}") resp = _post(base, path, body) if "_error" not in resp: model = resp.get("model_used", "") fallback = resp.get("is_fallback", True) output = resp.get("output", {}) all_pass &= _check("Model is real", _is_real_model(model), f"model_used={model}") all_pass &= _check("Not fallback", not fallback, f"is_fallback={fallback}") all_pass &= _check("Has output", bool(output), f"keys={list(output.keys())[:5]}") else: all_pass &= _check(f"{name} reachable", False, resp["_error"]) print() # Summary print("=" * 50) if all_pass: print("RESULT: ALL CHECKS PASSED — Real AI mode confirmed") print("DocDoe is generating real AI answers via OpenRouter.") else: print("RESULT: SOME CHECKS FAILED — Mock/fallback detected") print("Check AI_PROVIDER, OPENROUTER_API_KEY, and backend logs.") print() sys.exit(0 if all_pass else 1) if __name__ == "__main__": main()