backend/scripts/robust_backend.py

#!/usr/bin/env python3
"""
ATOM Robust Backend API Server
Production-ready backend with process management, auto-recovery, and comprehensive monitoring
"""

import asyncio
from contextlib import asynccontextmanager
import logging
import os
import signal
import sys
import time
import traceback
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
import psutil
from pydantic import BaseModel
import uvicorn

# Configure robust logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - [%(process)d] - %(message)s",
    handlers=[
        logging.StreamHandler(sys.stdout),
        logging.FileHandler("logs/backend_robust.log"),
    ],
)
logger = logging.getLogger(__name__)


# Global state for graceful management
class BackendState:
    def __init__(self):
        self.startup_time = time.time()
        self.healthy = False
        self.shutdown_event = asyncio.Event()
        self.restart_count = 0
        self.last_restart_time = 0


backend_state = BackendState()


# Signal handlers for graceful management
def signal_handler(signum, frame):
    """Handle shutdown signals gracefully"""
    logger.info(f"Received signal {signum}, initiating graceful shutdown...")
    backend_state.shutdown_event.set()


# Register signal handlers
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)


# Pydantic models for robust API
class HealthResponse(BaseModel):
    status: str
    service: str
    version: str
    timestamp: str
    message: str
    uptime: float
    process_id: int
    restart_count: int
    memory_usage_mb: float


class ServiceStatus(BaseModel):
    name: str
    status: str
    version: str
    endpoints: List[str]
    health: str


class IntegrationStatus(BaseModel):
    name: str
    status: str
    enabled: bool
    health_check: str
    last_check: str


class SystemStatusResponse(BaseModel):
    overall_status: str
    services: List[ServiceStatus]
    integrations: List[IntegrationStatus]
    uptime: float
    timestamp: str
    process_info: Dict
    system_metrics: Dict


class ProcessInfo(BaseModel):
    pid: int
    name: str
    status: str
    cpu_percent: float
    memory_mb: float
    threads: int
    uptime: float


@asynccontextmanager
async def lifespan(app: FastAPI):
    """Robust lifespan manager with error recovery"""
    # Startup
    logger.info("🚀 ATOM Robust Backend Starting Up...")
    startup_time = time.time()

    try:
        # Create necessary directories
        os.makedirs("logs", exist_ok=True)
        os.makedirs("data", exist_ok=True)
        os.makedirs("tmp", exist_ok=True)

        # Initialize services with retry logic
        await initialize_services_with_retry()

        backend_state.healthy = True
        backend_state.restart_count += 1
        backend_state.last_restart_time = time.time()

        logger.info("✅ ATOM Robust Backend Started Successfully")
        logger.info(f"📊 Process ID: {os.getpid()}")
        logger.info(f"⏰ Startup time: {time.time() - startup_time:.2f}s")

    except Exception as e:
        logger.error(f"❌ Startup failed: {e}")
        logger.error(traceback.format_exc())
        backend_state.healthy = False
        raise

    yield  # Application runs here

    # Shutdown
    try:
        logger.info("🛑 ATOM Robust Backend Shutting Down...")
        await shutdown_services()
        uptime = time.time() - startup_time
        logger.info(f"📊 Backend ran for {uptime:.2f} seconds")
        logger.info("👋 ATOM Robust Backend Shutdown Complete")
    except Exception as e:
        logger.error(f"❌ Shutdown error: {e}")


async def initialize_services_with_retry(max_retries: int = 3):
    """Initialize services with retry logic"""
    for attempt in range(max_retries):
        try:
            logger.info(
                f"Initializing services (attempt {attempt + 1}/{max_retries})..."
            )

            services = [
                "Authentication Service",
                "Database Connection Pool",
                "Integration Manager",
                "Task Queue System",
                "Cache Service",
                "File Storage",
                "Monitoring System",
            ]

            for service in services:
                logger.info(f"🔄 Initializing {service}...")
                await asyncio.sleep(0.1)  # Simulate initialization
                logger.info(f"✅ {service} initialized")

            logger.info("✅ All services initialized successfully")
            return

        except Exception as e:
            logger.warning(f"Service initialization attempt {attempt + 1} failed: {e}")
            if attempt < max_retries - 1:
                await asyncio.sleep(2**attempt)  # Exponential backoff
            else:
                raise


async def shutdown_services():
    """Gracefully shutdown all services"""
    logger.info("Shutting down services gracefully...")

    services = [
        "Database Connection Pool",
        "Task Queue System",
        "Cache Service",
        "Integration Manager",
        "File Storage",
    ]

    for service in services:
        try:
            logger.info(f"🛑 Shutting down {service}...")
            await asyncio.sleep(0.1)
            logger.info(f"✅ {service} shutdown complete")
        except Exception as e:
            logger.warning(f"Error shutting down {service}: {e}")


# Create FastAPI app with robust configuration
app = FastAPI(
    title="ATOM Robust Backend API",
    description="Advanced Task Orchestration & Management - Production-Ready API with Auto-Recovery",
    version="2.1.0-robust",
    docs_url="/docs",
    redoc_url="/redoc",
    lifespan=lifespan,
)

# Enhanced CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=[
        "http://localhost:3000",
        "http://127.0.0.1:3000",
        "http://localhost:3001",
        "http://127.0.0.1:3001",
        "http://localhost:4491",
        "http://127.0.0.1:4491",
        "http://localhost:8080",
        "http://127.0.0.1:8080",
    ],
    allow_credentials=True,
    allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"],
    allow_headers=["*"],
    expose_headers=["*"],
)


try:
    from backend.api.autoflow_routes import router as autoflow_router

    app.include_router(autoflow_router)
    logger.info("✓ Luuna Autoflow Core Routes Loaded")
except Exception as exc:
    logger.warning(f"Failed to load Luuna Autoflow Core routes: {exc}")


try:
    from backend.api.kingpdf_routes import router as kingpdf_router

    app.include_router(kingpdf_router)
    logger.info("✓ KingPDF Routes Loaded")
except Exception as exc:
    logger.warning(f"Failed to load KingPDF routes: {exc}")


# Global exception handler
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
    """Global exception handler with detailed logging"""
    logger.error(f"Unhandled exception in {request.method} {request.url}: {exc}")
    logger.error(traceback.format_exc())

    return JSONResponse(
        status_code=500,
        content={
            "ok": False,
            "error": {
                "code": "INTERNAL_SERVER_ERROR",
                "message": "An internal server error occurred",
                "request_id": str(hash(request)),
                "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
            },
        },
    )


# Enhanced health check endpoint
@app.get("/health", response_model=HealthResponse)
async def health_check():
    """Comprehensive health check with system metrics"""
    if not backend_state.healthy:
        raise HTTPException(status_code=503, detail="Service unhealthy")

    process = psutil.Process()
    memory_info = process.memory_info()

    return HealthResponse(
        status="healthy",
        service="atom-robust-backend",
        version="2.1.0",
        timestamp=time.strftime("%Y-%m-%d %H:%M:%S"),
        message="ATOM Robust Backend is running optimally",
        uptime=time.time() - backend_state.startup_time,
        process_id=os.getpid(),
        restart_count=backend_state.restart_count,
        memory_usage_mb=memory_info.rss / 1024 / 1024,
    )


@app.get("/healthz")
async def healthz():
    """Lightweight local health check alias."""
    return {
        "ok": True,
        "status": "healthy" if backend_state.healthy else "starting",
        "service": "atom-robust-backend",
        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
    }


@app.get("/api/documents")
async def list_documents():
    """Local development fallback for the documents page."""
    return {"success": True, "data": [], "source": "local-fallback"}


LOCAL_AGENTS: List[Dict[str, Any]] = [
    {
        "id": "local-research-agent",
        "name": "Research Agent",
        "description": "Local fallback agent for research and synthesis.",
        "status": "idle",
        "last_run": None,
        "category": "research",
    },
    {
        "id": "local-workflow-agent",
        "name": "Workflow Agent",
        "description": "Local fallback agent for workflow planning.",
        "status": "idle",
        "last_run": None,
        "category": "automation",
    },
]


@app.get("/api/agents")
@app.get("/api/agents/")
async def list_agents(category: Optional[str] = None):
    """Local development fallback for agent registry."""
    if category:
        return [agent for agent in LOCAL_AGENTS if agent["category"] == category]
    return LOCAL_AGENTS


@app.get("/api/analytics/dashboard/kpis")
async def analytics_dashboard_kpis():
    """Stable local analytics fallback."""
    return {
        "success": True,
        "source": "local-fallback",
        "kpis": {
            "documents": 0,
            "agents": len(LOCAL_AGENTS),
            "workflow_executions": 0,
            "active_integrations": 0,
        },
    }


@app.get("/api/workflow-templates")
@app.get("/api/workflow-templates/")
async def workflow_templates():
    """Stable local workflow template fallback."""
    return []


@app.get("/api/marketing/dashboard/summary")
async def marketing_dashboard_summary():
    """Stable local marketing dashboard fallback."""
    return {
        "success": True,
        "source": "local-fallback",
        "summary": {
            "campaigns": 0,
            "leads": 0,
            "conversions": 0,
            "spend": 0,
        },
    }


@app.get("/api/v1/workflow-ui/executions")
@app.get("/api/workflows/executions")
async def workflow_executions():
    """Stable JSON fallback for workflow executions."""
    return {"success": True, "executions": [], "source": "local-fallback"}


@app.get("/api/v1/workflow-ui/services")
@app.get("/api/workflows/services")
async def workflow_services():
    return {"success": True, "services": {}, "source": "local-fallback"}


@app.get("/api/v1/workflow-ui/definitions")
@app.get("/api/workflows/definitions")
async def workflow_definitions():
    return {"success": True, "workflows": [], "source": "local-fallback"}


@app.get("/ws/stats")
async def websocket_stats():
    """Stable local WebSocket stats fallback."""
    return {
        "success": True,
        "source": "local-fallback",
        "connections": 0,
        "active_channels": 0,
        "messages_sent": 0,
    }


# Root endpoint with comprehensive info
@app.get("/")
async def root():
    """Root endpoint with detailed system information"""
    process = psutil.Process()
    uptime = time.time() - backend_state.startup_time

    return {
        "name": "ATOM Robust Backend API",
        "status": "running" if backend_state.healthy else "unhealthy",
        "version": "2.1.0",
        "uptime": f"{uptime:.2f} seconds",
        "process_id": os.getpid(),
        "restart_count": backend_state.restart_count,
        "system": {
            "python_version": sys.version,
            "platform": sys.platform,
            "working_directory": os.getcwd(),
        },
        "endpoints": {
            "health": "/health",
            "system_status": "/api/system/status",
            "integrations": "/api/integrations/status",
            "process_info": "/api/process/info",
            "docs": "/docs",
        },
        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
    }


# Comprehensive system status endpoint
@app.get("/api/system/status", response_model=SystemStatusResponse)
async def system_status():
    """Detailed system status with metrics"""
    process = psutil.Process()
    system_metrics = {
        "cpu_percent": psutil.cpu_percent(),
        "memory_percent": psutil.virtual_memory().percent,
        "disk_usage": psutil.disk_usage(".").percent,
    }

    services = [
        ServiceStatus(
            name="Backend API",
            status="running",
            version="2.1.0",
            endpoints=["/health", "/api/system/status", "/api/integrations/status"],
            health="healthy",
        ),
        ServiceStatus(
            name="Database",
            status="ready",
            version="1.0.0",
            endpoints=["/api/data/*"],
            health="healthy",
        ),
        ServiceStatus(
            name="Authentication",
            status="ready",
            version="1.0.0",
            endpoints=["/api/auth/*"],
            health="healthy",
        ),
        ServiceStatus(
            name="Integration Manager",
            status="running",
            version="1.0.0",
            endpoints=["/api/integrations/*"],
            health="healthy",
        ),
    ]

    integrations = [
        IntegrationStatus(
            name="Asana",
            status="available",
            enabled=True,
            health_check="/api/integrations/asana/health",
            last_check=time.strftime("%Y-%m-%d %H:%M:%S"),
        ),
        IntegrationStatus(
            name="Slack",
            status="available",
            enabled=True,
            health_check="/api/integrations/slack/health",
            last_check=time.strftime("%Y-%m-%d %H:%M:%S"),
        ),
        IntegrationStatus(
            name="GitHub",
            status="available",
            enabled=True,
            health_check="/api/integrations/github/health",
            last_check=time.strftime("%Y-%m-%d %H:%M:%S"),
        ),
        IntegrationStatus(
            name="Notion",
            status="available",
            enabled=True,
            health_check="/api/integrations/notion/health",
            last_check=time.strftime("%Y-%m-%d %H:%M:%S"),
        ),
        IntegrationStatus(
            name="Jira",
            status="available",
            enabled=True,
            health_check="/api/integrations/jira/health",
            last_check=time.strftime("%Y-%m-%d %H:%M:%S"),
        ),
    ]

    return SystemStatusResponse(
        overall_status="healthy",
        services=services,
        integrations=integrations,
        uptime=time.time() - backend_state.startup_time,
        timestamp=time.strftime("%Y-%m-%d %H:%M:%S"),
        process_info={
            "pid": process.pid,
            "name": process.name(),
            "status": process.status(),
            "cpu_percent": process.cpu_percent(),
            "memory_mb": process.memory_info().rss / 1024 / 1024,
            "threads": process.num_threads(),
        },
        system_metrics=system_metrics,
    )


# Integration status endpoint
@app.get("/api/integrations/status")
async def integrations_status():
    """Integration status with availability checks"""
    integrations = [
        {
            "name": "Asana",
            "status": "ready",
            "endpoints": ["/api/asana/health", "/api/auth/asana/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
        {
            "name": "Slack",
            "status": "ready",
            "endpoints": ["/api/slack/health", "/api/auth/slack/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
        {
            "name": "GitHub",
            "status": "ready",
            "endpoints": ["/api/github/health", "/api/auth/github/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
        {
            "name": "Notion",
            "status": "ready",
            "endpoints": ["/api/notion/health", "/api/auth/notion/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
        {
            "name": "Jira",
            "status": "ready",
            "endpoints": ["/api/jira/health", "/api/auth/jira/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
        {
            "name": "Trello",
            "status": "ready",
            "endpoints": ["/api/trello/health", "/api/auth/trello/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
        {
            "name": "Google Workspace",
            "status": "ready",
            "endpoints": ["/api/google/health", "/api/auth/google/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
        {
            "name": "Microsoft 365",
            "status": "ready",
            "endpoints": ["/api/microsoft/health", "/api/auth/microsoft/authorize"],
            "health": "healthy",
            "needs_oauth": True,
        },
    ]

    total_integrations = len(integrations)
    available_integrations = len([i for i in integrations if i["health"] == "healthy"])
    success_rate = (available_integrations / total_integrations) * 100

    return {
        "ok": True,
        "integrations": integrations,
        "total_integrations": total_integrations,
        "available_integrations": available_integrations,
        "success_rate": f"{success_rate:.1f}%",
        "message": f"{available_integrations}/{total_integrations} integrations available and ready for OAuth configuration",
        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
    }


# Process information endpoint
@app.get("/api/process/info", response_model=ProcessInfo)
async def process_info():
    """Detailed process information"""
    process = psutil.Process()

    return ProcessInfo(
        pid=process.pid,
        name=process.name(),
        status=process.status(),
        cpu_percent=process.cpu_percent(),
        memory_mb=process.memory_info().rss / 1024 / 1024,
        threads=process.num_threads(),
        uptime=time.time() - backend_state.startup_time,
    )


# Mock integration health endpoints
@app.get("/api/asana/health")
async def asana_health():
    return {
        "ok": True,
        "service": "asana",
        "status": "ready",
        "message": "Asana integration is ready for OAuth configuration",
        "needs_oauth": True,
        "endpoints": {
            "authorize": "/api/auth/asana/authorize",
            "callback": "/api/auth/asana/callback",
            "search": "/api/asana/search",
            "list_tasks": "/api/asana/list-tasks",
        },
    }


@app.get("/api/slack/health")
async def slack_health():
    return {
        "ok": True,
        "service": "slack",
        "status": "ready",
        "message": "Slack integration is ready for OAuth configuration",
        "needs_oauth": True,
    }


@app.get("/api/github/health")
async def github_health():
    return {
        "ok": True,
        "service": "github",
        "status": "ready",
        "message": "GitHub integration is ready for OAuth configuration",
        "needs_oauth": True,
    }


# Graceful shutdown endpoint (protected)
@app.post("/api/shutdown")
async def graceful_shutdown():
    """Initiate graceful shutdown (requires authentication in production)"""
    # In production, this would require proper authentication
    logger.info("Graceful shutdown initiated via API")
    backend_state.shutdown_event.set()
    return {
        "ok": True,
        "message": "Shutdown initiated",
        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
    }


# System metrics endpoint
@app.get("/api/metrics")
async def system_metrics():
    """System metrics for monitoring"""
    process = psutil.Process()

    return {
        "process": {
            "pid": process.pid,
            "name": process.name(),
            "status": process.status(),
            "cpu_percent": process.cpu_percent(),
            "memory_mb": process.memory_info().rss / 1024 / 1024,
            "threads": process.num_threads(),
            "uptime": time.time() - backend_state.startup_time,
        },
        "system": {
            "cpu_percent": psutil.cpu_percent(),
            "memory_percent": psutil.virtual_memory().percent,
            "disk_usage": psutil.disk_usage(".").percent,
        },
    }