Spaces:
Sleeping
Sleeping
File size: 3,920 Bytes
1bb4678 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
# Path: QAgents-workflos/tests/fast_eval.py
# Fast evaluation - one problem per difficulty, all modes
"""Fast mode evaluation."""
import sys
import os
import time
import json
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.absolute()))
api_key = "$env:GOOGLE_API_KEY"
os.environ['GOOGLE_API_KEY'] = api_key
from tests.test_problems import (
PROBLEM_E1_PHASE_FLIP,
PROBLEM_M1_SWAP_DECOMPOSITION,
PROBLEM_H1_DEUTSCH,
PROBLEM_VH4_BERNSTEIN_VAZIRANI
)
from orchestrators import create_orchestrator
from orchestrators.quasar_orchestrator import QuasarOrchestrator, HybridOrchestrator
from config import set_api_key
import re
set_api_key(api_key)
def extract_gates(qasm):
if not qasm:
return 0
gate_pattern = r'\b(h|x|y|z|s|t|cx|cz|swap|ccx|rz|rx|ry|cp)\b'
return len(re.findall(gate_pattern, qasm, re.IGNORECASE))
def test_problem(problem, mode, timeout=60):
start = time.perf_counter()
try:
if mode == "quasar":
orch = QuasarOrchestrator(max_iterations=3)
result = orch.run(problem.prompt, problem.expected.min_qubits)
return {"success": result.success, "time_ms": (time.perf_counter()-start)*1000,
"llm": result.llm_calls, "gates": extract_gates(result.final_qasm), "error": None}
elif mode == "hybrid":
orch = HybridOrchestrator()
result = orch.run(problem.prompt, problem.expected.min_qubits)
return {"success": result.success, "time_ms": (time.perf_counter()-start)*1000,
"llm": result.llm_calls, "gates": extract_gates(result.final_qasm), "error": None}
else:
orch = create_orchestrator(mode)
result = orch.run(problem.prompt)
llm = 1 if mode == "naked" else len(result.agent_results) if result.agent_results else 0
return {"success": result.success, "time_ms": (time.perf_counter()-start)*1000,
"llm": llm, "gates": extract_gates(result.final_output), "error": "; ".join(result.errors) if result.errors else None}
except Exception as e:
return {"success": False, "time_ms": (time.perf_counter()-start)*1000,
"llm": 0, "gates": 0, "error": str(e)[:60]}
print("=" * 70)
print("FAST MODE EVALUATION")
print("=" * 70)
print(f"Date: {datetime.now().isoformat()}")
problems = [
("EASY", PROBLEM_E1_PHASE_FLIP),
("MEDIUM", PROBLEM_M1_SWAP_DECOMPOSITION),
("HARD", PROBLEM_H1_DEUTSCH),
("VERY_HARD", PROBLEM_VH4_BERNSTEIN_VAZIRANI)
]
modes = ["naked", "quasar", "hybrid", "blackboard"]
all_results = {}
for diff, problem in problems:
print(f"\n{diff}: {problem.name}")
print("-" * 50)
all_results[diff] = {}
for mode in modes:
print(f" {mode:12}", end=" ", flush=True)
result = test_problem(problem, mode)
all_results[diff][mode] = result
status = "✅" if result["success"] else "❌"
print(f"{status} {result['time_ms']:5.0f}ms LLM:{result['llm']} Gates:{result['gates']}")
if result["error"]:
print(f" ⚠️ {result['error'][:40]}...")
time.sleep(5)
# Summary
print("\n" + "=" * 70)
print("SUMMARY")
print("=" * 70)
for mode in modes:
successes = sum(1 for diff in all_results if all_results[diff][mode]["success"])
total_time = sum(all_results[diff][mode]["time_ms"] for diff in all_results)
total_llm = sum(all_results[diff][mode]["llm"] for diff in all_results)
print(f"\n{mode.upper():12} {successes}/4 ({25*successes}%) | {total_time:.0f}ms | {total_llm} LLM calls")
for diff in all_results:
r = all_results[diff][mode]
status = "✅" if r["success"] else "❌"
print(f" {diff:10} {status}")
print("\n" + "=" * 70)
print("DONE")
|