tests/run_golden_set.py

# tests/run_golden_set.py — V1.0 (Integration Test Runner)
import asyncio
import os
import sys
import json
import logging
import time

# Add parent dir to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Import core components
try:
    from orchestrator import BuddyOrchestrator
except ImportError as e:
    print(f"❌ Error importing orchestrator: {e}")
    sys.exit(1)

# Configure logging to be minimal for the runner
logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger("GOLDEN_SET")

GOLDEN_SET_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "qa_golden_set")

async def run_single_test(orchestrator, filename):
    filepath = os.path.join(GOLDEN_SET_DIR, filename)
    print(f"🔬 Testing: {filename}...", end=" ", flush=True)
    
    with open(filepath, "rb") as f:
        image_data = f.read()
    
    start_time = time.time()
    try:
        # V277.0: FIXED - solve_problem is now an async generator. 
        # We must iterate over it and collect the final COMPLETE payload.
        final_result = {}
        async for event in orchestrator.solve_problem(
            problem_text="[GOLDEN_SET_AUTO_OCR]",
            grade="י'",
            student_name="QA_BOT",
            image_data=image_data
        ):
            # V8.5 logic: If the state is COMPLETE, the payload contains the full standard response
            if event.state == "COMPLETE" and event.payload:
                final_result = event.payload
            elif event.state == "ERROR":
                final_result = event.payload or {"logic_error": True, "status": "ERROR"}

        result = final_result
        duration = time.time() - start_time
        
        status = result.get("status", "SUCCESS")
        logic_error = result.get("logic_error", False)
        error_type = result.get("error_type", "NONE")
        
        icon = "✅"
        if logic_error:
            icon = f"❌ LOGIC_ERROR ({error_type})"
        elif not result:
            icon = "🛑 EMPTY_RESULT"
            logic_error = True
        elif status != "SUCCESS":
            icon = f"⚠️ {status}"
        else:
            # 🚨 [SOP FIX] Ensure sections are not empty on Happy Path
            sections = result.get("sections", [])
            if not sections:
                icon = "⚠️ EMPTY_SECTIONS"
                # We don't necessarily fail the whole test if it's a simple answer only, 
                # but for the Golden Set, we expect sections.
            
        print(f"{icon} ({duration:.1f}s)")
        
        return {
            "file": filename,
            "status": status,
            "logic_error": logic_error,
            "error_type": error_type,
            "duration": round(duration, 1),
            "final_answer": str(result.get("final_answer", ""))[:100] + "..."
        }
    except Exception as e:
        print(f"💥 CRASH: {e}")
        return {
            "file": filename,
            "status": "CRASH",
            "logic_error": True,
            "error_type": "CRASH",
            "duration": 0,
            "final_answer": str(e)
        }

async def main():
    if not os.path.exists(GOLDEN_SET_DIR):
        print(f"❌ Directory not found: {GOLDEN_SET_DIR}")
        return

    orchestrator = BuddyOrchestrator()
    files = [f for f in os.listdir(GOLDEN_SET_DIR) if f.lower().endswith(('.jpg', '.jpeg', '.png'))]
    files.sort()
    
    print(f"\n🚀 Starting QA Golden Set Validation (10 images)")
    print("=" * 60)
    
    results = []
    for f in files:
        res = await run_single_test(orchestrator, f)
        results.append(res)
        # Sleep briefly between tests to avoid hitting quotas too hard
        await asyncio.sleep(2)
    
    print("=" * 60)
    print(f"🏁 Finished. Success: {len([r for r in results if not r['logic_error']])}/{len(results)}")
    
    # Save a JSON report
    report_path = os.path.join(os.path.dirname(GOLDEN_SET_DIR), "temp", "golden_set_report.json")
    os.makedirs(os.path.dirname(report_path), exist_ok=True)
    with open(report_path, "w", encoding="utf-8") as rf:
        json.dump(results, rf, indent=2, ensure_ascii=False)
    print(f"📊 Report saved to: {report_path}\n")

if __name__ == "__main__":
    asyncio.run(main())