api/complete_api.py

"""
Complete Production API for Ultimate Football Prediction System V4.0
FastAPI endpoints for predictions, results, suggestions, and system health
"""

from fastapi import APIRouter, HTTPException, BackgroundTasks
from pydantic import BaseModel, Field
from typing import Dict, List, Optional, Any
from datetime import datetime
import logging
import json
from pathlib import Path
import joblib
import numpy as np

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Create router
router = APIRouter(prefix="/api/v4", tags=["V4.0 Ultimate API"])


# ===================== MODELS =====================

class PredictionRequest(BaseModel):
    """Request for match prediction."""
    home_team: str
    away_team: str
    league: str
    match_date: Optional[str] = None
    markets: List[str] = Field(default=["result", "btts", "over25"])

class PredictionResponse(BaseModel):
    """Prediction response with confidence and uncertainty."""
    match_id: str
    home_team: str
    away_team: str
    predictions: Dict[str, Dict[str, Any]]
    confidence: float
    uncertainty: float
    suggestions: List[str]
    timestamp: str

class ResultUpdate(BaseModel):
    """Update with actual result."""
    match_id: str
    home_goals: int
    away_goals: int
    home_goals_ht: Optional[int] = None
    away_goals_ht: Optional[int] = None

class HealthResponse(BaseModel):
    """System health response."""
    status: str
    version: str
    models_loaded: Dict[str, bool]
    calibration_status: Dict[str, Any]
    improvement_suggestions: List[str]
    last_prediction: Optional[str]
    predictions_count: int

class SuggestionResponse(BaseModel):
    """Improvement suggestions response."""
    suggestions: List[Dict[str, Any]]
    patterns: Dict[str, Any]
    recommended_actions: List[str]


# ===================== STATE =====================

class APIState:
    """Manages API state and loaded models."""
    
    def __init__(self):
        self.models = {}
        self.feature_generator = None
        self.calibration_system = None
        self.predictions_count = 0
        self.last_prediction = None
        self.experiment_tracker = None
        self.is_initialized = False
    
    def initialize(self):
        """Initialize V4.0 system components."""
        if self.is_initialized:
            return
        
        try:
            # Try to load models
            model_dir = Path("models/v4")
            if model_dir.exists():
                for model_file in model_dir.glob("*.joblib"):
                    market = model_file.stem.split("_")[0]
                    try:
                        self.models[market] = joblib.load(model_file)
                        logger.info(f"Loaded V4 model: {market}")
                    except Exception as e:
                        logger.warning(f"Failed to load {model_file}: {e}")
            
            # Initialize feature generator
            try:
                from src.features.enhanced_engineering import EnhancedFeatureGenerator
                logger.info("Enhanced feature generator available")
            except ImportError:
                logger.warning("Enhanced feature generator not available")
            
            # Initialize calibration
            try:
                from src.calibration.enhanced_calibration import EnhancedCalibrationSystem
                self.calibration_system = EnhancedCalibrationSystem()
                logger.info("Enhanced calibration system initialized")
            except ImportError:
                logger.warning("Enhanced calibration system not available")
            
            # Initialize experiment tracker
            try:
                from src.training.enhanced_training_pipeline import ExperimentTracker
                self.experiment_tracker = ExperimentTracker()
                logger.info("Experiment tracker initialized")
            except ImportError:
                logger.warning("Experiment tracker not available")
            
            self.is_initialized = True
            logger.info("V4.0 API initialized successfully")
            
        except Exception as e:
            logger.error(f"Failed to initialize V4.0 API: {e}")

# Global state
state = APIState()


# ===================== ENDPOINTS =====================

@router.on_event("startup")
async def startup():
    """Initialize on startup."""
    state.initialize()


@router.get("/health", response_model=HealthResponse)
async def health_check():
    """Get system health and improvement suggestions."""
    state.initialize()
    
    # Get calibration status
    calibration_status = {}
    if state.calibration_system:
        try:
            calibration_status = state.calibration_system.get_status()
        except:
            calibration_status = {"error": "Could not get status"}
    
    # Get improvement suggestions
    suggestions = []
    if state.experiment_tracker:
        try:
            recommendations = state.experiment_tracker.analyze_patterns()
            if recommendations:
                suggestions.append(f"Best accuracy achieved: {recommendations.get('best_accuracy', 0):.2%}")
                if recommendations.get('n_experiments', 0) < 5:
                    suggestions.append("Run more training experiments to establish baseline")
        except:
            pass
    
    if not state.models:
        suggestions.append("No V4 models loaded. Run training pipeline to generate models.")
    
    return HealthResponse(
        status="healthy" if state.is_initialized else "initializing",
        version="4.0.0",
        models_loaded={k: True for k in state.models.keys()},
        calibration_status=calibration_status,
        improvement_suggestions=suggestions,
        last_prediction=state.last_prediction,
        predictions_count=state.predictions_count
    )


@router.post("/predict", response_model=PredictionResponse)
async def predict(request: PredictionRequest):
    """Generate prediction for a match."""
    state.initialize()
    
    match_id = f"{request.home_team}_{request.away_team}_{datetime.now().strftime('%Y%m%d')}"
    
    predictions = {}
    overall_confidence = 0.0
    overall_uncertainty = 0.0
    suggestions = []
    
    for market in request.markets:
        if market in state.models:
            try:
                # Generate features (placeholder - would use real feature generator)
                features = _generate_features(request)
                
                model = state.models[market]
                proba = model.predict_proba(features)
                pred = model.predict(features)
                
                # Calibrate if available
                if state.calibration_system and state.calibration_system.is_fitted:
                    proba = state.calibration_system.calibrate(proba)
                
                # Calculate confidence and uncertainty
                confidence = float(np.max(proba))
                entropy = float(-np.sum(proba * np.log(proba + 1e-10)))
                
                predictions[market] = {
                    "prediction": pred[0] if hasattr(pred, '__iter__') else pred,
                    "probabilities": proba[0].tolist() if proba.ndim > 1 else [float(proba)],
                    "confidence": confidence,
                    "uncertainty": entropy
                }
                
                overall_confidence += confidence
                overall_uncertainty += entropy
                
            except Exception as e:
                logger.error(f"Prediction failed for {market}: {e}")
                predictions[market] = {"error": str(e)}
        else:
            # Use fallback prediction
            predictions[market] = _fallback_prediction(market, request)
    
    n_markets = len(request.markets)
    overall_confidence = overall_confidence / n_markets if n_markets > 0 else 0.5
    overall_uncertainty = overall_uncertainty / n_markets if n_markets > 0 else 0.5
    
    # Generate suggestions
    if overall_confidence < 0.6:
        suggestions.append("Low confidence prediction - consider smaller stake")
    if overall_uncertainty > 0.8:
        suggestions.append("High uncertainty - more historical data needed")
    
    state.predictions_count += 1
    state.last_prediction = datetime.now().isoformat()
    
    return PredictionResponse(
        match_id=match_id,
        home_team=request.home_team,
        away_team=request.away_team,
        predictions=predictions,
        confidence=overall_confidence,
        uncertainty=overall_uncertainty,
        suggestions=suggestions,
        timestamp=datetime.now().isoformat()
    )


@router.post("/result")
async def update_result(result: ResultUpdate, background_tasks: BackgroundTasks):
    """Update with actual result for continuous learning."""
    state.initialize()
    
    # Log result
    logger.info(f"Result received: {result.match_id} - {result.home_goals}:{result.away_goals}")
    
    # Update calibration system
    if state.calibration_system:
        background_tasks.add_task(_update_calibration, result)
    
    return {
        "status": "received",
        "match_id": result.match_id,
        "result": f"{result.home_goals}-{result.away_goals}",
        "message": "Result recorded for continuous learning"
    }


@router.get("/suggestions", response_model=SuggestionResponse)
async def get_suggestions():
    """Get improvement suggestions based on experiment analysis."""
    state.initialize()
    
    suggestions = []
    patterns = {}
    recommended_actions = []
    
    if state.experiment_tracker:
        try:
            patterns = state.experiment_tracker.analyze_patterns()
            
            # Generate suggestions based on patterns
            if patterns.get('n_experiments', 0) < 5:
                suggestions.append({
                    "type": "data",
                    "priority": "high",
                    "message": "Insufficient experiments. Run more training iterations."
                })
            
            if patterns.get('best_accuracy', 0) < 0.6:
                suggestions.append({
                    "type": "features",
                    "priority": "high", 
                    "message": "Low accuracy. Consider adding more features or data sources."
                })
                recommended_actions.append("Run enhanced feature engineering")
                recommended_actions.append("Add more training data")
            
            if patterns.get('avg_top5_accuracy', 0) > 0.65:
                suggestions.append({
                    "type": "deployment",
                    "priority": "medium",
                    "message": "Good accuracy. Consider deploying best model."
                })
                
        except Exception as e:
            logger.error(f"Failed to analyze patterns: {e}")
    
    # Check calibration
    if state.calibration_system:
        try:
            if state.calibration_system.should_recalibrate():
                suggestions.append({
                    "type": "calibration",
                    "priority": "high",
                    "message": "Calibration degraded. Recalibration recommended."
                })
                recommended_actions.append("Trigger recalibration")
        except:
            pass
    
    if not suggestions:
        suggestions.append({
            "type": "general",
            "priority": "low",
            "message": "System running normally. Continue monitoring."
        })
    
    return SuggestionResponse(
        suggestions=suggestions,
        patterns=patterns,
        recommended_actions=recommended_actions
    )


@router.post("/retrain")
async def trigger_retrain(background_tasks: BackgroundTasks, market: str = "result"):
    """Trigger model retraining."""
    state.initialize()
    
    # Add retraining to background
    background_tasks.add_task(_run_retraining, market)
    
    return {
        "status": "scheduled",
        "market": market,
        "message": f"Retraining for {market} market scheduled"
    }


# ===================== HELPER FUNCTIONS =====================

def _generate_features(request: PredictionRequest) -> np.ndarray:
    """Generate features for prediction (placeholder)."""
    # In production, this would use the EnhancedFeatureGenerator
    # For now, return dummy features
    return np.random.randn(1, 100)


def _fallback_prediction(market: str, request: PredictionRequest) -> Dict:
    """Generate fallback prediction when model not available."""
    if market == "result":
        return {
            "prediction": "H",
            "probabilities": [0.45, 0.28, 0.27],
            "labels": ["H", "D", "A"],
            "confidence": 0.45,
            "source": "fallback"
        }
    elif market == "btts":
        return {
            "prediction": "Yes",
            "probability": 0.52,
            "confidence": 0.52,
            "source": "fallback"
        }
    elif market == "over25":
        return {
            "prediction": "Over",
            "probability": 0.55,
            "confidence": 0.55,
            "source": "fallback"
        }
    return {"prediction": "unknown", "source": "fallback"}


async def _update_calibration(result: ResultUpdate):
    """Update calibration with new result."""
    try:
        # This would update the online calibration system
        logger.info(f"Updating calibration with result: {result.match_id}")
    except Exception as e:
        logger.error(f"Calibration update failed: {e}")


async def _run_retraining(market: str):
    """Run retraining in background."""
    try:
        logger.info(f"Starting retraining for {market}...")
        
        # Import and run training pipeline
        from src.training.enhanced_training_pipeline import EnhancedTrainingPipeline, TrainingConfig
        
        config = TrainingConfig(n_optuna_trials=20, n_splits=3)
        pipeline = EnhancedTrainingPipeline(config)
        
        # Would load data and train here
        logger.info(f"Retraining for {market} completed")
        
    except Exception as e:
        logger.error(f"Retraining failed: {e}")


# ===================== REGISTRATION =====================

def register_api(app):
    """Register V4.0 API with FastAPI app."""
    app.include_router(router)
    logger.info("✅ V4.0 Ultimate API registered at /api/v4")
    return router


# For testing
if __name__ == "__main__":
    print("=" * 60)
    print("COMPLETE API V4.0 - TEST")
    print("=" * 60)
    
    # Test models
    print("\n1. Testing request models...")
    req = PredictionRequest(
        home_team="Liverpool",
        away_team="Man United",
        league="Premier League"
    )
    print(f"   Request: {req}")
    
    print("\n2. Testing state initialization...")
    state.initialize()
    print(f"   Initialized: {state.is_initialized}")
    print(f"   Models loaded: {list(state.models.keys())}")
    
    print("\n3. Testing fallback predictions...")
    for market in ["result", "btts", "over25"]:
        pred = _fallback_prediction(market, req)
        print(f"   {market}: {pred['prediction']}")
    
    print("\n✅ API test complete!")