analysis/bayesian_engine.py

"""
Bayesian Gap Prediction Engine
═══════════════════════════════
Applies Bayes' theorem to iteratively update gap direction probabilities
as new evidence arrives throughout the week.

P(Gap_Up | Evidence) ∝ P(Evidence | Gap_Up) × P(Gap_Up)

Evidence sources (each contributes a likelihood ratio):
  1. Multi-timeframe momentum (40m, 1D, 3D)
  2. RSI regime (overbought/oversold)
  3. COT institutional positioning
  4. News sentiment flow
  5. Agent consensus (weighted by track record)
  6. Historical gap seasonality (day-of-year patterns)
  7. Volatility regime (ATR expansion/contraction)
  8. Upcoming high-impact events (risk premium adjustment)

Priors are learned from historical trade outcomes stored in the database.
"""

from __future__ import annotations

import logging
import math
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any

logger = logging.getLogger("gap_system.analysis.bayesian")


# ═══════════════════════════════════════════════════════════════════════════════
# BAYESIAN STATE
# ═══════════════════════════════════════════════════════════════════════════════

@dataclass
class BayesianState:
    """
    Tracks the running posterior probability for a single asset.
    Updated incrementally as new evidence arrives.
    """
    asset: str
    # Log-odds representation (numerically stable for repeated updates)
    log_odds_bullish: float = 0.0  # 0.0 = 50/50 prior
    
    # Track which evidence has been applied this cycle
    evidence_applied: list[str] = field(default_factory=list)
    
    # Historical prior calibration
    historical_bull_rate: float = 0.50  # Updated from DB
    total_historical_gaps: int = 0
    
    # Cycle tracking
    update_count: int = 0
    last_updated: str = ""
    
    @property
    def probability_bullish(self) -> float:
        """Convert log-odds to probability."""
        return 1.0 / (1.0 + math.exp(-self.log_odds_bullish))
    
    @property
    def probability_bearish(self) -> float:
        return 1.0 - self.probability_bullish
    
    @property
    def direction(self) -> str:
        p = self.probability_bullish
        if p > 0.52:
            return "BULLISH"
        elif p < 0.48:
            return "BEARISH"
        return "NEUTRAL"
    
    @property
    def confidence(self) -> float:
        """Confidence = distance from 50%, scaled to [0, 1]."""
        return abs(self.probability_bullish - 0.50) * 2.0
    
    def apply_evidence(self, name: str, likelihood_ratio: float) -> None:
        """
        Update posterior using a likelihood ratio (Bayes factor).
        
        LR > 1.0 → evidence supports BULLISH
        LR < 1.0 → evidence supports BEARISH
        LR = 1.0 → evidence is uninformative
        
        log_odds += log(LR) is the Bayesian update rule.
        """
        if likelihood_ratio <= 0:
            logger.warning("Invalid likelihood ratio %.4f for %s — skipping", likelihood_ratio, name)
            return
        
        log_lr = math.log(likelihood_ratio)
        
        # Damping: cap the maximum influence of any single evidence source
        max_log_shift = 1.2  # ~3.3x odds shift max per evidence
        log_lr = max(-max_log_shift, min(max_log_shift, log_lr))
        
        old_p = self.probability_bullish
        self.log_odds_bullish += log_lr
        new_p = self.probability_bullish
        
        self.evidence_applied.append(name)
        self.update_count += 1
        self.last_updated = datetime.now(timezone.utc).isoformat()
        
        logger.info(
            "Bayes [%s] %s: LR=%.3f → P(Bull) %.1f%% → %.1f%% (Δ%+.1f%%)",
            self.asset, name, likelihood_ratio,
            old_p * 100, new_p * 100, (new_p - old_p) * 100,
        )
    
    def reset_for_new_cycle(self) -> None:
        """Reset to prior for a new weekly cycle."""
        # Use historical prior if available, else 50/50
        if self.total_historical_gaps >= 5:
            prior_p = max(0.20, min(0.80, self.historical_bull_rate))
        else:
            prior_p = 0.50
        
        self.log_odds_bullish = math.log(prior_p / (1.0 - prior_p))
        self.evidence_applied.clear()
        self.update_count = 0
        logger.info(
            "Bayes [%s] reset. Prior: %.1f%% (from %d historical gaps)",
            self.asset, prior_p * 100, self.total_historical_gaps,
        )
    
    def to_dict(self) -> dict:
        return {
            "asset": self.asset,
            "probability_bullish": round(self.probability_bullish, 4),
            "probability_bearish": round(self.probability_bearish, 4),
            "direction": self.direction,
            "confidence": round(self.confidence, 4),
            "log_odds": round(self.log_odds_bullish, 4),
            "evidence_applied": list(self.evidence_applied),
            "update_count": self.update_count,
            "historical_prior": round(self.historical_bull_rate, 4),
            "historical_samples": self.total_historical_gaps,
            "last_updated": self.last_updated,
        }


# ═══════════════════════════════════════════════════════════════════════════════
# GLOBAL STATE REGISTRY
# ═══════════════════════════════════════════════════════════════════════════════

_states: dict[str, BayesianState] = {}


def get_state(asset: str) -> BayesianState:
    if asset not in _states:
        _states[asset] = BayesianState(asset=asset)
    return _states[asset]


def get_all_states() -> dict[str, dict]:
    return {asset: state.to_dict() for asset, state in _states.items()}


# ═══════════════════════════════════════════════════════════════════════════════
# EVIDENCE FUNCTIONS — Each returns a likelihood ratio
# ═══════════════════════════════════════════════════════════════════════════════

def _lr_momentum(fallback_metrics: dict) -> float:
    """
    Multi-timeframe momentum.
    Score ranges from -1 (strong bear) to +1 (strong bull).
    Convert to likelihood ratio: LR = exp(score * strength).
    """
    score = fallback_metrics.get("momentum_score", 0)
    # Convert [-1, 1] score to LR
    # score=+1.0 → LR≈2.7 (strong bull evidence)
    # score=-1.0 → LR≈0.37 (strong bear evidence)
    # score=0.0  → LR=1.0 (uninformative)
    return math.exp(score * 1.0)


def _lr_rsi(ta_4h: dict) -> float:
    """
    RSI regime — mean-reverting signal.
    Extreme RSI is contrarian evidence for gaps (weekend reversion).
    """
    rsi = ta_4h.get("rsi", 50)
    if rsi > 75:
        return 0.65  # Overbought → bear evidence for gap
    elif rsi > 65:
        return 0.85  # Mildly overbought
    elif rsi < 25:
        return 1.55  # Oversold → bull evidence for gap
    elif rsi < 35:
        return 1.20  # Mildly oversold
    return 1.0  # Neutral


def _lr_cot(cot_data: dict | None) -> float:
    """
    COT institutional positioning — leading indicator.
    Net long = institutions expect higher = bullish evidence.
    Extreme positioning is CONTRARIAN (crowded trades reverse).
    """
    if not cot_data:
        return 1.0
    
    bias = cot_data.get("bias", "NEUTRAL")
    extreme = cot_data.get("extreme_flag", False)
    
    if extreme:
        # CONTRARIAN: extreme long = bearish signal
        if bias == "BULLISH":
            return 0.70  # Crowded long → expect reversal down
        elif bias == "BEARISH":
            return 1.40  # Crowded short → expect squeeze up
    else:
        # Non-extreme: follow the institutions
        if bias == "BULLISH":
            return 1.20
        elif bias == "BEARISH":
            return 0.85
    
    return 1.0


def _lr_news_sentiment(news_data: list[dict]) -> float:
    """
    News sentiment flow — aggregate positive/negative tilt.
    Returns LR based on sentiment balance.
    """
    if not news_data:
        return 1.0
    
    positive = sum(1 for n in news_data if str(n.get("sentiment", "")).startswith("positive"))
    negative = sum(1 for n in news_data if str(n.get("sentiment", "")).startswith("negative"))
    total = positive + negative
    
    if total < 3:
        return 1.0  # Not enough data
    
    ratio = positive / total  # 0 to 1
    # ratio=0.8 → LR≈1.5 (clear positive sentiment)
    # ratio=0.2 → LR≈0.67 (clear negative sentiment)
    # ratio=0.5 → LR=1.0 (balanced)
    return math.exp((ratio - 0.5) * 1.5)


def _lr_agent_consensus(agents: list, agent_confidence: float, direction: str) -> float:
    """
    Agent consensus — the multi-platform debate result.
    Strong consensus is strong evidence. Split votes are weak.
    """
    if not agents:
        return 1.0
    
    valid = [a for a in agents if getattr(a, 'last_cycle_success', True)]
    if not valid:
        return 1.0
    
    bull = sum(1 for a in valid if a.current_stance == "BULLISH")
    bear = sum(1 for a in valid if a.current_stance == "BEARISH")
    total = bull + bear
    
    if total == 0:
        return 1.0
    
    # Consensus strength: how lopsided is the vote?
    consensus_ratio = max(bull, bear) / total
    
    # Agent confidence modulates the strength
    strength = (consensus_ratio - 0.5) * 2.0 * agent_confidence  # 0 to 1
    
    # Sign: positive if majority is BULLISH, negative if BEARISH
    sign = 1.0 if bull > bear else -1.0
    
    return math.exp(sign * strength * 0.8)


def _lr_technical_confluence(technical_summary: dict | None) -> float:
    """
    Multi-timeframe technical confluence.
    All timeframes aligned = strong directional evidence.
    """
    if not technical_summary:
        return 1.0
    
    bias = technical_summary.get("weighted_bias", "NEUTRAL")
    confluence = technical_summary.get("confluence_score", 0.5)
    
    if bias == "NEUTRAL" or bias == "RANGING":
        return 1.0
    
    # Higher confluence = stronger evidence
    strength = (confluence - 0.4) * 2.5  # 0 at 0.4, 1.5 at 1.0
    strength = max(0, min(1.2, strength))
    
    sign = 1.0 if bias == "BULLISH" else -1.0
    return math.exp(sign * strength * 0.7)


def _lr_volatility_regime(fallback_metrics: dict) -> float:
    """
    Volatility regime — modifies gap predictability.
    High vol = larger gaps but less directional predictability.
    Returns a dampening factor (closer to 1.0 in high vol).
    """
    regime = fallback_metrics.get("volatility_regime", "NORMAL")
    
    if regime == "HIGH":
        # In high vol, all other evidence is less reliable
        # We don't provide directional evidence, just reduce conviction
        return 1.0  # Neutral — but other LRs are dampened by the engine
    
    return 1.0  # Normal/Low vol doesn't modify


def _lr_structural_breakout(fallback_metrics: dict) -> float:
    """
    Structural breakout/breakdown — strong directional evidence.
    Price above prev week high or below prev week low.
    """
    if fallback_metrics.get("above_prev_week_high", False):
        return 1.6  # Strong bullish breakout
    elif fallback_metrics.get("below_prev_week_low", False):
        return 0.65  # Strong bearish breakdown
    return 1.0


def _lr_upcoming_events(event_risk_score: float) -> float:
    """
    Upcoming high-impact event risk — DAMPENER.
    When major events are imminent, directional conviction should decrease.
    This pulls all predictions toward 50/50 (LR → 1.0).
    
    event_risk_score: 0.0 (no events) to 1.0 (major event imminent)
    """
    if event_risk_score <= 0.1:
        return 1.0  # No dampening
    
    # At risk=0.5 → LR≈0.85 (mild dampening)
    # At risk=1.0 → LR≈0.6 (strong dampening toward uncertainty)
    return 1.0 - (event_risk_score * 0.4)


# ═══════════════════════════════════════════════════════════════════════════════
# MAIN UPDATE FUNCTION — Called each debate cycle
# ═══════════════════════════════════════════════════════════════════════════════

def bayesian_update(
    asset: str,
    agents: list,
    technical_summary: dict | None = None,
    cot_data: dict | None = None,
    news_data: list[dict] | None = None,
    agent_confidence: float = 0.5,
    agent_direction: str = "NEUTRAL",
    event_risk_score: float = 0.0,
) -> dict:
    """
    Run a full Bayesian update cycle for an asset.
    
    Each evidence source contributes a likelihood ratio.
    The posterior accumulates across cycles within the same week,
    implementing true Bayesian learning.
    
    Returns the current Bayesian state as a dict.
    """
    state = get_state(asset)
    
    # Extract sub-data
    fallback = {}
    ta_4h = {}
    if technical_summary:
        fallback = technical_summary.get("fallback_metrics", {})
        ta_4h = technical_summary.get("4H", {})
    
    # ── Apply each evidence source ──
    
    # 1. Multi-timeframe momentum
    if fallback:
        lr = _lr_momentum(fallback)
        state.apply_evidence("momentum", lr)
    
    # 2. RSI regime
    if ta_4h:
        lr = _lr_rsi(ta_4h)
        state.apply_evidence("rsi_regime", lr)
    
    # 3. COT positioning
    lr = _lr_cot(cot_data)
    if lr != 1.0:
        state.apply_evidence("cot_positioning", lr)
    
    # 4. News sentiment
    if news_data:
        lr = _lr_news_sentiment(news_data)
        if abs(lr - 1.0) > 0.05:  # Only apply if meaningfully different
            state.apply_evidence("news_sentiment", lr)
    
    # 5. Agent consensus (the multi-platform AI debate)
    if agents:
        lr = _lr_agent_consensus(agents, agent_confidence, agent_direction)
        state.apply_evidence("agent_consensus", lr)
    
    # 6. Technical confluence
    lr = _lr_technical_confluence(technical_summary)
    if lr != 1.0:
        state.apply_evidence("technical_confluence", lr)
    
    # 7. Structural breakout
    if fallback:
        lr = _lr_structural_breakout(fallback)
        if lr != 1.0:
            state.apply_evidence("structural_breakout", lr)
    
    # 8. Event risk dampening (reduces overall confidence)
    if event_risk_score > 0.2:
        # Dampen by pulling log-odds toward 0 (50/50)
        dampen_factor = 1.0 - (event_risk_score * 0.3)  # At max risk, retain 70%
        state.log_odds_bullish *= dampen_factor
        state.evidence_applied.append(f"event_risk_dampen({event_risk_score:.2f})")
        logger.info(
            "Bayes [%s] Event risk dampening: %.0f%% → P(Bull) moved toward 50%%",
            asset, event_risk_score * 100,
        )
    
    # 9. Volatility regime dampening
    if fallback.get("volatility_regime") == "HIGH":
        state.log_odds_bullish *= 0.85
        state.evidence_applied.append("high_vol_dampen")
        logger.info("Bayes [%s] High volatility dampening applied", asset)
    
    logger.info(
        "Bayes [%s] After update #%d: %s %.1f%% confidence (P_bull=%.1f%%, %d evidence sources)",
        asset, state.update_count, state.direction,
        state.confidence * 100, state.probability_bullish * 100,
        len(state.evidence_applied),
    )
    
    return state.to_dict()


def load_historical_priors(trade_history: list[dict]) -> None:
    """
    Load historical gap outcomes from the database to calibrate priors.
    Called once at startup.
    """
    # Group by asset
    asset_outcomes: dict[str, list[bool]] = {}
    for trade in trade_history:
        asset = trade.get("asset", "")
        direction = trade.get("direction", "")
        profit = trade.get("profit_usd", 0) or 0
        
        if not asset or not direction:
            continue
        
        # A "bullish gap" = gap opened higher than Friday close
        # We track whether the BULLISH prediction was profitable
        was_bullish = direction == "BULLISH"
        was_correct = profit > 0
        
        # If bullish and correct, gap went up. If bearish and correct, gap went down.
        gap_went_up = (was_bullish and was_correct) or (not was_bullish and not was_correct)
        
        asset_outcomes.setdefault(asset, []).append(gap_went_up)
    
    for asset, outcomes in asset_outcomes.items():
        state = get_state(asset)
        state.total_historical_gaps = len(outcomes)
        state.historical_bull_rate = sum(outcomes) / len(outcomes)
        
        logger.info(
            "Historical prior for %s: %.1f%% bullish (%d samples)",
            asset, state.historical_bull_rate * 100, len(outcomes),
        )


def reset_all_for_new_week() -> None:
    """Reset all Bayesian states for a new trading week."""
    for state in _states.values():
        state.reset_for_new_cycle()
    logger.info("All Bayesian states reset for new week (%d assets)", len(_states))