#!/usr/bin/env python3 """ Smoke test: 3 cases (easy/medium/hard) to verify all models work. Shows optimal path vs what each model gets + timing. """ from __future__ import annotations import os import sys import time # Fix Windows console encoding if sys.platform == "win32": sys.stdout.reconfigure(encoding="utf-8", errors="replace") # Suppress warnings os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0" import warnings warnings.filterwarnings("ignore") from pathlib import Path project_root = Path(__file__).parent.parent sys.path.insert(0, str(project_root)) import logging logging.basicConfig(level=logging.WARNING) from src.agents import get_agent from src.game import GameEngine # ============================================================================= # 3 TEST CASES # ============================================================================= TEST_CASES = [ ("Easy", "Espresso", "Leonardo da Vinci", "3-5 clicks expected"), ("Medium", "IKEA", "Cleopatra", "8-12 clicks expected"), ("Hard", "Fortnite", "Bubonic plague", "10-20 clicks expected"), ("Tricky", "Jeffrey Epstein", "Severna Park, Maryland", "unknown"), ("Tricky", "Severna Park, Maryland", "Jeffrey Epstein", "unknown"), ] # ============================================================================= # MODELS TO TEST # ============================================================================= LLM_MODELS = [ # === SMALL/FAST MODELS (2026) === "openai/gpt-4o-mini", # Budget baseline "openai/gpt-5-mini", # GPT-5 small "deepseek/deepseek-chat", # DeepSeek budget "google/gemini-2.0-flash-lite-001", # Gemini 2.0 lite "google/gemini-3-flash-preview", # Gemini 3 flash "anthropic/claude-haiku-4.5", # Claude 4.5 haiku # === BIG/SMART MODELS (2026) === "openai/gpt-5.2-chat", # GPT-5.2 (Dec 2025) "deepseek/deepseek-v3.2", # DeepSeek V3.2 "google/gemini-3-pro-preview", # Gemini 3 Pro (Nov 2025) "anthropic/claude-sonnet-4.5", # Claude Sonnet 4.5 ] def run_game(agent_name: str, start: str, target: str, max_steps: int = 30, **kwargs) -> dict: """Run a single game and return result dict.""" try: agent = get_agent(agent_name, **kwargs) with GameEngine(visualize=False) as engine: start_time = time.time() result = engine.run(agent=agent, start=start, target=target, max_steps=max_steps) elapsed = time.time() - start_time # Get LLM stats if available stats = agent.get_stats() if hasattr(agent, "get_stats") else {} fallbacks = stats.get("fallback_count", 0) llm_requests = stats.get("total_requests", 0) return { "agent": agent.name, "won": result.won, "clicks": result.total_clicks, "time": round(elapsed, 2), "path": result.path, "llm_requests": llm_requests, "fallbacks": fallbacks, "used_fallback": fallbacks > 0, } except Exception as e: return { "agent": agent_name, "won": False, "clicks": -1, "time": 0, "path": [], "error": str(e)[:80], } def main(): print("=" * 90) print("SMOKE TEST: 5 Cases × 10 LLM Models (Small vs Big)") print("=" * 90) print("Comparing small/fast models vs big/smart models (Jan 2026)") print() all_results = {} for difficulty, start, target, expected in TEST_CASES: print("=" * 90) print(f"{difficulty.upper()}: {start} → {target}") print(f"Expected: {expected}") print("=" * 90) case_key = f"{start} → {target}" all_results[case_key] = {"difficulty": difficulty, "results": []} print(f"\n{'Agent':<40} {'Result':>8} {'Clicks':>7} {'LLM?':>6} {'Time':>8}") print("-" * 75) # Run each LLM model for model in LLM_MODELS: model_short = model.split("/")[-1][:38] result = run_game("llm", start, target, model=model) all_results[case_key]["results"].append(result) if "error" in result: print(f"{model_short:<40} {'ERROR':>8} {'-':>7} {'-':>6} {'-':>8}") print(f" Error: {result['error'][:60]}") else: status = "WIN" if result["won"] else "LOST" clicks_str = str(result["clicks"]) if result["won"] else ">30" # Show if result is from LLM or fallback llm_indicator = "Yes" if not result.get("used_fallback", False) else "FALL" print(f"{model_short:<40} {status:>8} {clicks_str:>7} {llm_indicator:>6} {result['time']:>7.1f}s") if result.get("used_fallback"): print(f" (Used embedding fallback - LLM rate limited)") # Small delay to avoid rate limits time.sleep(1.0) print() # ========================================================================== # SUMMARY # ========================================================================== print("\n" + "=" * 90) print("SUMMARY BY MODEL") print("=" * 90) # Aggregate results by agent agent_stats = {} for case_key, case_data in all_results.items(): for r in case_data["results"]: agent = r["agent"] if agent not in agent_stats: agent_stats[agent] = {"wins": 0, "total": 0, "clicks": [], "times": []} agent_stats[agent]["total"] += 1 if r["won"]: agent_stats[agent]["wins"] += 1 agent_stats[agent]["clicks"].append(r["clicks"]) agent_stats[agent]["times"].append(r["time"]) print(f"\n{'Agent':<45} {'Wins':>8} {'AvgClicks':>10} {'AvgTime':>10}") print("-" * 75) # Sort by wins, then avg clicks sorted_agents = sorted( agent_stats.items(), key=lambda x: (-x[1]["wins"], sum(x[1]["clicks"])/len(x[1]["clicks"]) if x[1]["clicks"] else 999) ) for agent, stats in sorted_agents: wins = f"{stats['wins']}/{stats['total']}" avg_clicks = f"{sum(stats['clicks'])/len(stats['clicks']):.1f}" if stats['clicks'] else "-" avg_time = f"{sum(stats['times'])/len(stats['times']):.1f}s" if stats['times'] else "-" print(f"{agent:<45} {wins:>8} {avg_clicks:>10} {avg_time:>10}") # ========================================================================== # TOP 5 RECOMMENDATIONS # ========================================================================== print("\n" + "=" * 90) print("TOP 5 LLM RECOMMENDATIONS") print("=" * 90) llm_agents = [(a, s) for a, s in sorted_agents if "llm-" in a] for i, (agent, stats) in enumerate(llm_agents[:5], 1): model_name = agent.replace("llm-", "") wins = stats["wins"] avg_clicks = sum(stats['clicks'])/len(stats['clicks']) if stats['clicks'] else 0 avg_time = sum(stats['times'])/len(stats['times']) if stats['times'] else 0 print(f"{i}. {model_name}") print(f" Wins: {wins}/{len(TEST_CASES)}, Avg clicks: {avg_clicks:.1f}, Avg time: {avg_time:.1f}s") # ========================================================================== # SAVE RESULTS # ========================================================================== import json from datetime import datetime output_file = project_root / "data" / "smoke_test_results.json" output_file.parent.mkdir(exist_ok=True) results_data = { "timestamp": datetime.now().isoformat(), "test_cases": [{"difficulty": d, "start": s, "target": t} for d, s, t, _ in TEST_CASES], "models": LLM_MODELS, "results": all_results, "summary": { agent: { "wins": stats["wins"], "total": stats["total"], "avg_clicks": sum(stats['clicks'])/len(stats['clicks']) if stats['clicks'] else None, "avg_time": sum(stats['times'])/len(stats['times']) if stats['times'] else None, } for agent, stats in agent_stats.items() } } with open(output_file, "w") as f: json.dump(results_data, f, indent=2) print(f"\nResults saved to: {output_file}") return 0 if __name__ == "__main__": sys.exit(main())