ComputerAssignments/CA10_knowledge_graphs/integrations/main_execution_advanced.py

#!/usr/bin/env python3
"""
CA10 Knowledge Graphs - Advanced Main Execution Script
This script runs all components of the Knowledge Graphs project with LangGraph and RAG
"""

import os
import sys
import matplotlib.pyplot as plt
import matplotlib
from datetime import datetime
import warnings
import asyncio

warnings.filterwarnings("ignore")

# Set matplotlib backend for saving figures
matplotlib.use("Agg")

# Add src to path
sys.path.append(os.path.join(os.path.dirname(__file__), "src"))

# Import all necessary modules
from core.knowledge_graph import KnowledgeGraph, create_sample_kg
from core.scientific_kg import AdvancedScientificKG, create_advanced_scientific_kg
from models.embeddings import TransE, DistMult, ComplEx, KGEmbeddingTrainer

# Import advanced components
try:
    from reasoning.langgraph_reasoning import (
        KnowledgeGraphReasoner,
        MultiAgentReasoningSystem,
    )
    from reasoning.rag_system import KnowledgeGraphRAG
    from api.advanced_api import AdvancedKnowledgeGraphAPI

    ADVANCED_FEATURES = True
    print("✅ Advanced features loaded successfully")
except ImportError as e:
    print(f"⚠️ Advanced features not available: {e}")
    ADVANCED_FEATURES = False


def setup_directories():
    """Create necessary directories"""
    dirs = ["logs", "results", "visualization", "src/api"]
    for dir_name in dirs:
        os.makedirs(dir_name, exist_ok=True)
    print("✅ Directories created successfully")


def save_figure(fig, filename, title=""):
    """Save figure to visualization directory"""
    if os.getenv("SAVE_VISUALIZATIONS", "false").lower() == "true":
        viz_dir = os.getenv("VISUALIZATION_DIR", "visualization")
        filepath = os.path.join(viz_dir, filename)
        fig.savefig(filepath, dpi=300, bbox_inches="tight")
        print(f"💾 Saved visualization: {filepath}")
        if title:
            print(f"   Title: {title}")
    plt.close(fig)


def run_basic_knowledge_graph():
    """Run basic knowledge graph creation and visualization"""
    print("\n🔬 Running Basic Knowledge Graph Analysis")
    print("=" * 50)

    # Create sample knowledge graph
    kg = create_sample_kg()
    kg.print_stats()

    # Visualize basic KG
    fig = plt.figure(figsize=(15, 10))
    kg.visualize(max_entities=15, layout="spring")
    plt.title(
        "🔬 Basic Knowledge Graph Visualization\nScientists and Their Discoveries",
        fontsize=14,
        fontweight="bold",
    )
    plt.tight_layout()
    save_figure(fig, "basic_knowledge_graph.png", "Basic Knowledge Graph")

    return kg


def run_advanced_scientific_kg():
    """Run advanced scientific knowledge graph analysis"""
    print("\n🧪 Running Advanced Scientific Knowledge Graph Analysis")
    print("=" * 60)

    # Create advanced scientific KG
    scientific_kg = create_advanced_scientific_kg()

    # Print statistics
    print(f"Advanced Scientific KG Statistics:")
    print(f"   • Total Entities: {len(scientific_kg.entities)}")
    print(f"   • Scientific Entities: {len(scientific_kg.scientific_entities)}")
    print(f"   • Total Triples: {len(scientific_kg.triples)}")

    # Analyze temporal patterns
    temporal_data = scientific_kg.analyze_temporal_patterns()
    print(f"   • Temporal Events: {len(temporal_data['years'])} years")

    # Calculate network metrics
    metrics = scientific_kg.calculate_network_metrics()
    print(
        f"   • Influence Network Nodes: {len(metrics.get('influence', {}).get('centrality', {}))}"
    )
    print(
        f"   • Collaboration Network Nodes: {len(metrics.get('collaboration', {}).get('centrality', {}))}"
    )

    # Find influential entities
    influential = scientific_kg.find_influential_entities(5)
    print(f"\n🌟 Top 5 Most Influential Entities:")
    for i, (entity_id, score) in enumerate(influential, 1):
        entity_name = scientific_kg.entities[entity_id].name
        print(f"   {i}. {entity_name} (Score: {score:.3f})")

    # Detect collaboration clusters
    clusters = scientific_kg.detect_collaboration_clusters()
    print(f"\n🤝 Collaboration Clusters: {len(clusters)}")
    for i, cluster in enumerate(clusters[:3], 1):  # Show top 3 clusters
        cluster_names = [scientific_kg.entities[eid].name for eid in cluster]
        print(f"   Cluster {i}: {', '.join(cluster_names)}")

    # Visualize advanced network
    scientific_kg.visualize_advanced_network(layout="spring", max_entities=30)
    fig = plt.gcf()
    save_figure(
        fig, "advanced_scientific_network.png", "Advanced Scientific Network Analysis"
    )

    # Export analysis
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    analysis_file = scientific_kg.export_analysis(
        f"results/scientific_kg_analysis_{timestamp}.json"
    )

    return scientific_kg


def run_embedding_models():
    """Run knowledge graph embedding models"""
    print("\n🧠 Running Knowledge Graph Embedding Models")
    print("=" * 50)

    # Create knowledge graph for embeddings
    kg = create_sample_kg()

    # Test different embedding models
    models = {
        "TransE": TransE(len(kg.entities), len(kg.relations), embedding_dim=50),
        "DistMult": DistMult(len(kg.entities), len(kg.relations), embedding_dim=50),
        "ComplEx": ComplEx(len(kg.entities), len(kg.relations), embedding_dim=50),
    }

    results = {}

    for model_name, model in models.items():
        print(f"\n📊 Training {model_name} model...")
        trainer = KGEmbeddingTrainer(model, kg)

        # Train the model
        losses = trainer.train(epochs=50, lr=0.001, batch_size=64)

        # Evaluate the model
        eval_results = trainer.evaluate_link_prediction(k=3)
        results[model_name] = eval_results

        print(f"   Hits@3: {eval_results.get('hits@k', 0):.3f}")
        print(f"   Mean Rank: {eval_results.get('mean_rank', 0):.2f}")

        # Visualize embeddings
        embeddings_2d = trainer.visualize_embeddings(method="pca", max_entities=30)
        fig = plt.gcf()
        save_figure(
            fig,
            f"{model_name.lower()}_embeddings.png",
            f"{model_name} Entity Embeddings",
        )

        # Find similar entities
        if len(kg.entities) > 0:
            sample_entity = list(kg.entities.keys())[0]
            similar = trainer.find_similar_entities(sample_entity, k=5)
            print(f"   Similar to '{kg.entities[sample_entity].name}':")
            for entity_id, similarity in similar:
                entity_name = kg.entities[entity_id].name
                print(f"     • {entity_name} (similarity: {similarity:.3f})")

    # Compare models
    print(f"\n📈 Model Comparison:")
    for model_name, result in results.items():
        print(
            f"   {model_name}: Hits@3={result.get('hits@k', 0):.3f}, Mean Rank={result.get('mean_rank', 0):.2f}"
        )

    return results


def run_advanced_reasoning():
    """Run advanced reasoning with LangGraph"""
    if not ADVANCED_FEATURES:
        print("\n⚠️ Advanced reasoning skipped (features not available)")
        return None

    print("\n🧠 Running Advanced Reasoning with LangGraph")
    print("=" * 50)

    # Create knowledge graph
    kg = create_sample_kg()

    # Initialize reasoning systems
    print("🔧 Initializing LangGraph reasoning systems...")
    reasoner = KnowledgeGraphReasoner(kg)
    multi_agent = MultiAgentReasoningSystem(kg)

    # Test queries
    test_queries = [
        "What did Einstein discover?",
        "Who influenced Newton?",
        "What are the relationships between physics concepts?",
        "Find all scientists who worked on gravity",
    ]

    async def run_reasoning_tests():
        results = {}

        for query in test_queries:
            print(f"\n🔍 Testing query: {query}")

            # Single agent reasoning
            try:
                result = await reasoner.reason(query)
                print(f"   Single Agent Answer: {result['answer'][:100]}...")
                print(f"   Confidence: {result['confidence']:.3f}")
                results[f"{query}_single"] = result
            except Exception as e:
                print(f"   Single Agent Error: {e}")

            # Multi-agent reasoning
            try:
                result = await multi_agent.reason(query)
                print(f"   Multi-Agent Answer: {result['answer'][:100]}...")
                print(f"   Confidence: {result['confidence']:.3f}")
                results[f"{query}_multi"] = result
            except Exception as e:
                print(f"   Multi-Agent Error: {e}")

        return results

    # Run async reasoning tests
    try:
        reasoning_results = asyncio.run(run_reasoning_tests())

        # Save results
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        results_file = f"results/reasoning_results_{timestamp}.json"

        import json

        with open(results_file, "w") as f:
            json.dump(reasoning_results, f, indent=2, default=str)

        print(f"\n💾 Reasoning results saved to: {results_file}")

        return reasoning_results

    except Exception as e:
        print(f"❌ Advanced reasoning failed: {e}")
        return None


def run_rag_system():
    """Run RAG system for knowledge graph"""
    if not ADVANCED_FEATURES:
        print("\n⚠️ RAG system skipped (features not available)")
        return None

    print("\n🔍 Running RAG System for Knowledge Graph")
    print("=" * 50)

    # Create knowledge graph
    kg = create_sample_kg()

    # Initialize RAG system
    print("🔧 Initializing RAG system...")
    rag_system = KnowledgeGraphRAG(kg)

    # Test queries
    test_queries = [
        "What discoveries did Einstein make?",
        "Who are the most influential scientists?",
        "What is the relationship between physics and mathematics?",
        "Find all works written by Darwin",
    ]

    async def run_rag_tests():
        results = {}

        for query in test_queries:
            print(f"\n🔍 RAG Query: {query}")

            try:
                result = await rag_system.query(query, k=5)
                print(f"   Answer: {result.answer[:100]}...")
                print(f"   Confidence: {result.confidence:.3f}")
                print(f"   Entities Found: {len(result.entities_found)}")
                print(f"   Relations Found: {len(result.relations_found)}")

                results[query] = {
                    "answer": result.answer,
                    "confidence": result.confidence,
                    "entities_found": result.entities_found,
                    "relations_found": result.relations_found,
                    "reasoning_path": result.reasoning_path,
                }

            except Exception as e:
                print(f"   RAG Error: {e}")

        return results

    # Run async RAG tests
    try:
        rag_results = asyncio.run(run_rag_tests())

        # Save results
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        results_file = f"results/rag_results_{timestamp}.json"

        import json

        with open(results_file, "w") as f:
            json.dump(rag_results, f, indent=2, default=str)

        print(f"\n💾 RAG results saved to: {results_file}")

        return rag_results

    except Exception as e:
        print(f"❌ RAG system failed: {e}")
        return None


def run_reasoning_examples():
    """Run knowledge graph reasoning examples"""
    print("\n🔍 Running Knowledge Graph Reasoning Examples")
    print("=" * 50)

    kg = create_sample_kg()

    # Example queries
    queries = [
        (
            "Find all entities related to Einstein",
            lambda: kg.query_triples(subject="einstein"),
        ),
        ("Find all discoveries", lambda: kg.query_triples(predicate="discovered")),
        (
            "Find all physics concepts",
            lambda: [t for t in kg.triples if t.object in ["relativity", "gravity"]],
        ),
    ]

    for query_name, query_func in queries:
        print(f"\n🔎 {query_name}:")
        results = query_func()
        for result in results[:5]:  # Show first 5 results
            print(f"   • {result}")
        if len(results) > 5:
            print(f"   ... and {len(results) - 5} more results")

    # Subgraph extraction
    print(f"\n🌐 Subgraph Analysis:")
    einstein_neighbors = kg.get_neighbors("einstein")
    if einstein_neighbors:
        subgraph = kg.get_subgraph(list(einstein_neighbors), max_hops=1)
        print(
            f"   Einstein's subgraph: {len(subgraph.entities)} entities, {len(subgraph.triples)} triples"
        )

        # Visualize subgraph
        fig = plt.figure(figsize=(12, 8))
        subgraph.visualize(max_entities=10, layout="spring")
        plt.title("Einstein's Knowledge Subgraph", fontsize=14, fontweight="bold")
        plt.tight_layout()
        save_figure(fig, "einstein_subgraph.png", "Einstein's Knowledge Subgraph")


def start_api_server():
    """Start the advanced API server"""
    if not ADVANCED_FEATURES:
        print("\n⚠️ API server skipped (features not available)")
        return None

    print("\n🚀 Starting Advanced API Server")
    print("=" * 40)

    # Create knowledge graph
    kg = create_sample_kg()

    # Create API
    api = AdvancedKnowledgeGraphAPI(kg)

    print("🌐 API server will start on http://localhost:8000")
    print("📚 API documentation: http://localhost:8000/docs")
    print("🖥️ Web interface: http://localhost:8000")
    print(
        "\n💡 To start the server, run: python -c 'from src.api.advanced_api import AdvancedKnowledgeGraphAPI; from src.core.knowledge_graph import create_sample_kg; api = AdvancedKnowledgeGraphAPI(create_sample_kg()); api.run()'"
    )

    return api


def generate_summary_report():
    """Generate a comprehensive summary report"""
    print("\n📋 Generating Comprehensive Summary Report")
    print("=" * 40)

    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    report_content = f"""
# CA10 Knowledge Graphs - Advanced Execution Report
Generated: {timestamp}

## Project Overview
This report summarizes the execution of the advanced CA10 Knowledge Graphs project, including:
- Basic Knowledge Graph creation and visualization
- Advanced Scientific Knowledge Graph analysis
- Knowledge Graph Embedding models (TransE, DistMult, ComplEx)
- Advanced reasoning with LangGraph
- RAG (Retrieval-Augmented Generation) system
- Multi-agent reasoning systems
- Advanced API with FastAPI
- Reasoning examples and subgraph analysis

## Advanced Features Status
- LangGraph Integration: {'✅ Available' if ADVANCED_FEATURES else '❌ Not Available'}
- RAG System: {'✅ Available' if ADVANCED_FEATURES else '❌ Not Available'}
- Multi-Agent Reasoning: {'✅ Available' if ADVANCED_FEATURES else '❌ Not Available'}
- Advanced API: {'✅ Available' if ADVANCED_FEATURES else '❌ Not Available'}

## Generated Files
### Visualizations:
- Basic Knowledge Graph visualization: basic_knowledge_graph.png
- Advanced Scientific Network analysis: advanced_scientific_network.png
- TransE embeddings: transe_embeddings.png
- DistMult embeddings: distmult_embeddings.png
- ComplEx embeddings: complex_embeddings.png
- Einstein's subgraph: einstein_subgraph.png

### Results:
- Scientific KG analysis: scientific_kg_analysis_*.json
- Reasoning results: reasoning_results_*.json (if advanced features available)
- RAG results: rag_results_*.json (if advanced features available)

## Key Findings
- Successfully created and analyzed multiple knowledge graph structures
- Implemented and trained three different embedding models
- {'Implemented advanced reasoning with LangGraph and multi-agent systems' if ADVANCED_FEATURES else 'Advanced reasoning features require additional dependencies'}
- {'Implemented RAG system for enhanced query capabilities' if ADVANCED_FEATURES else 'RAG system requires additional dependencies'}
- Performed reasoning queries and subgraph extraction
- Generated comprehensive visualizations for analysis

## Technical Details
- Python environment with PyTorch, NetworkX, and visualization libraries
- {'LangGraph for advanced reasoning workflows' if ADVANCED_FEATURES else 'Basic reasoning capabilities'}
- {'RAG system with vector search and retrieval' if ADVANCED_FEATURES else 'Basic query capabilities'}
- {'FastAPI for comprehensive API interface' if ADVANCED_FEATURES else 'Basic functionality'}
- All visualizations saved in high resolution (300 DPI)
- Results exported in JSON format for further analysis

## API Usage
{'The advanced API server provides comprehensive endpoints for:' if ADVANCED_FEATURES else 'API features require additional dependencies:'}
{'• Query endpoints with advanced reasoning' if ADVANCED_FEATURES else '• Basic query capabilities'}
{'• RAG-based query system' if ADVANCED_FEATURES else '• Limited query capabilities'}
{'• Entity and relation management' if ADVANCED_FEATURES else '• Basic entity management'}
{'• Visualization generation' if ADVANCED_FEATURES else '• Basic visualization'}
{'• Embedding training and similarity search' if ADVANCED_FEATURES else '• Basic embedding capabilities'}

## Next Steps
1. Install additional dependencies for advanced features
2. Configure API keys for LLM services (OpenAI, Anthropic)
3. Set up vector database for production use
4. Deploy API server for production use
5. Implement additional reasoning patterns
"""

    report_file = f"results/advanced_execution_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.md"
    with open(report_file, "w") as f:
        f.write(report_content)

    print(f"📄 Advanced report saved to: {report_file}")


def main():
    """Main execution function"""
    print("🎯 CA10 Knowledge Graphs - Advanced Execution")
    print("=" * 50)

    try:
        # Setup
        setup_directories()

        # Run all components
        basic_kg = run_basic_knowledge_graph()
        advanced_kg = run_advanced_scientific_kg()
        embedding_results = run_embedding_models()

        # Run advanced features if available
        reasoning_results = run_advanced_reasoning()
        rag_results = run_rag_system()

        # Run basic reasoning
        run_reasoning_examples()

        # Setup API server info
        api_server = start_api_server()

        # Generate comprehensive summary
        generate_summary_report()

        print("\n✅ All components executed successfully!")
        print("=" * 50)
        print("📁 Check the following directories for outputs:")
        print("   • visualization/ - All generated plots and figures")
        print("   • results/ - Analysis results and reports")
        print("   • logs/ - Execution logs")

        if ADVANCED_FEATURES:
            print("\n🚀 Advanced Features Available:")
            print("   • LangGraph reasoning workflows")
            print("   • RAG system with vector search")
            print("   • Multi-agent reasoning")
            print("   • Advanced API with FastAPI")
            print("\n🌐 To start the API server:")
            print(
                '   python -c "from src.api.advanced_api import AdvancedKnowledgeGraphAPI; from src.core.knowledge_graph import create_sample_kg; api = AdvancedKnowledgeGraphAPI(create_sample_kg()); api.run()"'
            )
        else:
            print("\n⚠️ Advanced features require additional dependencies:")
            print("   • Install: pip install langgraph langchain chromadb")
            print("   • Configure API keys for OpenAI/Anthropic")
            print("   • Run again to enable advanced features")

    except Exception as e:
        print(f"\n❌ Error during execution: {str(e)}")
        import traceback

        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()