system_verification.py

#!/usr/bin/env python3
"""
Comprehensive system verification script for the Agentic AI System.
Checks all components, dependencies, and connections.
"""

import sys
import importlib
from pathlib import Path
from typing import List, Dict, Any

# Add src to path
sys.path.insert(0, str(Path(__file__).parent / "src"))

def check_component(name: str, module_path: str, class_name: str = None) -> Dict[str, Any]:
    """Check if a component can be imported and instantiated."""
    try:
        module = importlib.import_module(module_path)
        
        if class_name:
            cls = getattr(module, class_name)
            # Try to instantiate (basic check)
            instance = cls()
            return {"status": "✅", "message": f"{name} working", "instance": instance}
        else:
            return {"status": "✅", "message": f"{name} module imported"}
            
    except ImportError as e:
        return {"status": "❌", "message": f"{name} import failed: {e}"}
    except Exception as e:
        return {"status": "⚠️", "message": f"{name} instantiation failed: {e}"}

def verify_system() -> None:
    """Perform comprehensive system verification."""
    
    print("🔍 Agentic AI System - Comprehensive Verification")
    print("=" * 60)
    
    # Core components to verify
    components = [
        # Configuration
        ("Settings", "config.settings", None),
        
        # Parsers
        ("SpaCy Parser", "src.parsers.spacy_parser", "SpacyParser"),
        ("Keyword Extractor", "src.parsers.keyword_extractor", "KeywordExtractor"),
        
        # API Clients
        ("API Matcher", "src.api_clients.api_matcher", "APIMatcher"),
        ("CDMS Client", "src.api_clients.cdms_client", "CDMSClient"),
        
        # Agents
        ("Base Agent", "src.agents.base_agent", None),
        ("Supervisor Agent", "src.agents.supervisor_agent", "SupervisorAgent"),
        ("Query Parser Agent", "src.agents.worker_agents", "QueryParserAgent"),
        ("API Matcher Agent", "src.agents.worker_agents", "APIMatcherAgent"),
        ("API Executor Agent", "src.agents.worker_agents", "APIExecutorAgent"),
        ("Result Formatter Agent", "src.agents.worker_agents", "ResultFormatterAgent"),
        
        # NEW: Evaluation Components
        ("Reflection Evaluator", "src.evaluators.reflection_evaluator", "ReflectionEvaluator"),
        ("Quality Assessor", "src.evaluators.quality_assessor", "QualityAssessor"),
        ("Memory Manager", "src.evaluators.memory_manager", "MemoryManager"),
        
        # NEW: RAG System
        ("Hybrid RAG System", "src.rag.hybrid_rag", "HybridRAGSystem"),
        
        # Utilities
        ("Logging Config", "src.utils.logging_config", None),
    ]
    
    print("\\n📦 Component Verification:")
    print("-" * 40)
    
    results = []
    for name, module_path, class_name in components:
        result = check_component(name, module_path, class_name)
        results.append((name, result))
        print(f"{result['status']} {name}: {result['message']}")
    
    # Check dependencies
    print("\\n🔧 Dependency Verification:")
    print("-" * 40)
    
    dependencies = [
        ("FastAPI", "fastapi"),
        ("Uvicorn", "uvicorn"),
        ("spaCy", "spacy"),
        ("LangChain", "langchain"),
        ("LangGraph", "langgraph"),
        ("KeyBERT", "keybert"),
        ("RapidFuzz", "rapidfuzz"),
        ("Sentence Transformers", "sentence_transformers"),
        ("Qdrant Client", "qdrant_client"),
        ("NumPy", "numpy"),
        ("Pydantic", "pydantic"),
        ("Requests", "requests"),
    ]
    
    dep_results = []
    for name, module in dependencies:
        try:
            importlib.import_module(module)
            print(f"✅ {name}: Available")
            dep_results.append((name, True))
        except ImportError:
            print(f"❌ {name}: Missing")
            dep_results.append((name, False))
    
    # Check spaCy model
    print("\\n🧠 NLP Model Verification:")
    print("-" * 40)
    
    try:
        import spacy
        nlp = spacy.load("en_core_web_sm")
        print("✅ spaCy en_core_web_sm: Available")
        spacy_model = True
    except OSError:
        print("❌ spaCy en_core_web_sm: Missing")
        print("   Run: python -m spacy download en_core_web_sm")
        spacy_model = False
    
    # Check file structure
    print("\\n📁 File Structure Verification:")
    print("-" * 40)
    
    required_files = [
        "api_server.py",
        "run_api.py", 
        "test_api_simple.py",
        "requirements.txt",
        "README.md",
        "config/settings.py",
        "src/__init__.py",
        "src/agents/__init__.py",
        "src/parsers/__init__.py",
        "src/api_clients/__init__.py",
        "src/evaluators/__init__.py",  # NEW
        "src/rag/__init__.py",         # NEW
        "src/utils/__init__.py",
        "tests/test_parsers.py",
        "tests/evaluation_harness.py", # NEW
    ]
    
    file_results = []
    for file_path in required_files:
        path = Path(file_path)
        if path.exists():
            print(f"✅ {file_path}: Exists")
            file_results.append((file_path, True))
        else:
            print(f"❌ {file_path}: Missing")
            file_results.append((file_path, False))
    
    # Test API server import
    print("\\n🚀 API Server Verification:")
    print("-" * 40)
    
    try:
        from api_server import app
        print("✅ API Server: Import successful")
        api_server = True
    except Exception as e:
        print(f"❌ API Server: Import failed - {e}")
        api_server = False
    
    # Test evaluation harness
    print("\\n🧪 Evaluation Harness Verification:")
    print("-" * 40)
    
    try:
        from tests.evaluation_harness import EvaluationHarness
        print("✅ Evaluation Harness: Import successful")
        eval_harness = True
    except Exception as e:
        print(f"❌ Evaluation Harness: Import failed - {e}")
        eval_harness = False
    
    # Summary
    print("\\n" + "=" * 60)
    print("📊 VERIFICATION SUMMARY")
    print("=" * 60)
    
    # Component summary
    component_success = len([r for _, r in results if r["status"] == "✅"])
    component_total = len(results)
    print(f"📦 Components: {component_success}/{component_total} working")
    
    # Dependency summary
    dep_success = len([r for _, r in dep_results if r])
    dep_total = len(dep_results)
    print(f"🔧 Dependencies: {dep_success}/{dep_total} available")
    
    # File summary
    file_success = len([r for _, r in file_results if r])
    file_total = len(file_results)
    print(f"📁 Files: {file_success}/{file_total} present")
    
    # Overall status
    overall_score = (component_success + dep_success + file_success) / (component_total + dep_total + file_total)
    
    print(f"\\n🎯 Overall System Health: {overall_score:.1%}")
    
    if overall_score >= 0.9:
        print("🎉 SYSTEM READY TO GO! ✨")
        print("\\n🚀 Quick Start:")
        print("   1. python run_api.py")
        print("   2. python test_api_simple.py")
        print("   3. python tests/evaluation_harness.py")
    elif overall_score >= 0.7:
        print("⚠️  SYSTEM MOSTLY READY - Minor issues detected")
        print("\\n💡 Check missing components above")
    else:
        print("❌ SYSTEM NOT READY - Major issues detected")
        print("\\n🔧 Install missing dependencies:")
        print("   pip install -r requirements.txt")
        if not spacy_model:
            print("   python -m spacy download en_core_web_sm")
    
    # Advanced features status
    print("\\n🌟 Advanced Features Status:")
    print("-" * 40)
    
    advanced_features = [
        ("Self-Critique Reflection", "src.evaluators.reflection_evaluator"),
        ("Persistent Memory", "src.evaluators.memory_manager"),
        ("Quality Assessment", "src.evaluators.quality_assessor"),
        ("Hybrid RAG System", "src.rag.hybrid_rag"),
        ("Evaluation Harness", "tests.evaluation_harness"),
    ]
    
    for name, module in advanced_features:
        try:
            importlib.import_module(module)
            print(f"✅ {name}: Available")
        except ImportError:
            print(f"❌ {name}: Not available")
    
    print("\\n" + "=" * 60)

if __name__ == "__main__":
    verify_system()