evolution/genetic_loop_v3.py

#!/usr/bin/env python3
"""
NBA Quant AI — REAL Genetic Evolution Loop v4
================================================
RUNS 24/7 on HF Space or Google Colab.

This is NOT a fake LLM wrapper. This is REAL ML:
  - Population of 500 individuals across 5 islands (100 per island)
  - 13 model types: tree-based + neural nets (LSTM, Transformer, TabNet, etc.)
  - NSGA-II Pareto front ranking (multi-objective: Brier, ROI, Sharpe, Calibration)
  - Island migration every 10 generations for diversity
  - Adaptive mutation: 0.15 -> 0.05 decay + stagnation boost
  - Memory management: GC between evaluations for 16GB RAM
  - Continuous cycles — saves after each generation
  - Callbacks to VM after each cycle
  - Population persistence (survives restarts)

Usage:
  # On HF Space (24/7):
  python evolution/genetic_loop_v3.py --continuous

  # On Google Colab (manual):
  !python genetic_loop_v3.py --generations 50

  # Quick test:
  python evolution/genetic_loop_v3.py --generations 5 --pop-size 50
"""

import os, sys, json, time, random, math, warnings, traceback, gc
import numpy as np
from pathlib import Path
from datetime import datetime, timezone, timedelta
from collections import defaultdict
from typing import Dict, List, Tuple, Optional

warnings.filterwarnings("ignore")

# All model types the GA can evolve
CPU_MODEL_TYPES = [
    "xgboost", "xgboost_brier", "lightgbm", "catboost", "random_forest", "extra_trees",
]
GPU_MODEL_TYPES = CPU_MODEL_TYPES + ["tabicl", "tabpfn"]
ALL_MODEL_TYPES = GPU_MODEL_TYPES + [
    "stacking", "mlp", "lstm", "transformer", "tabnet",
    "ft_transformer", "deep_ensemble", "autogluon",
]
NEURAL_NET_TYPES = {"lstm", "transformer", "tabnet", "ft_transformer", "deep_ensemble", "mlp", "autogluon"}
ICL_MODEL_TYPES = {"tabicl", "tabpfn"}  # In-context learning models (GPU, no hyperparams to tune)

# ── Run Logger (best-effort) ──
try:
    from evolution.run_logger import RunLogger
    _HAS_LOGGER = True
except ImportError:
    try:
        from run_logger import RunLogger
        _HAS_LOGGER = True
    except ImportError:
        _HAS_LOGGER = False

# ─── Auto-load .env.local ───
_env_file = Path(__file__).resolve().parent.parent / ".env.local"
if not _env_file.exists():
    _env_file = Path("/app/.env.local")
if _env_file.exists():
    for _line in _env_file.read_text().splitlines():
        _line = _line.strip()
        if _line and not _line.startswith("#") and "=" in _line:
            _line = _line.replace("export ", "")
            _k, _, _v = _line.partition("=")
            os.environ.setdefault(_k.strip(), _v.strip("'\""))

# ─── Paths ───
BASE_DIR = Path(__file__).resolve().parent.parent
DATA_DIR = BASE_DIR / "data"
HIST_DIR = DATA_DIR / "historical"
RESULTS_DIR = DATA_DIR / "results"
STATE_DIR = DATA_DIR / "evolution-state"
for d in [DATA_DIR, HIST_DIR, RESULTS_DIR, STATE_DIR]:
    d.mkdir(parents=True, exist_ok=True)

VM_CALLBACK_URL = os.environ.get("VM_CALLBACK_URL", "http://34.136.180.66:8080")
ODDS_API_KEY = os.environ.get("ODDS_API_KEY", "")


# ═══════════════════════════════════════════════════════════
# SECTION 1: DATA LOADING
# ═══════════════════════════════════════════════════════════

TEAM_MAP = {
    "Atlanta Hawks": "ATL", "Boston Celtics": "BOS", "Brooklyn Nets": "BKN",
    "Charlotte Hornets": "CHA", "Chicago Bulls": "CHI", "Cleveland Cavaliers": "CLE",
    "Dallas Mavericks": "DAL", "Denver Nuggets": "DEN", "Detroit Pistons": "DET",
    "Golden State Warriors": "GSW", "Houston Rockets": "HOU", "Indiana Pacers": "IND",
    "Los Angeles Clippers": "LAC", "Los Angeles Lakers": "LAL", "Memphis Grizzlies": "MEM",
    "Miami Heat": "MIA", "Milwaukee Bucks": "MIL", "Minnesota Timberwolves": "MIN",
    "New Orleans Pelicans": "NOP", "New York Knicks": "NYK", "Oklahoma City Thunder": "OKC",
    "Orlando Magic": "ORL", "Philadelphia 76ers": "PHI", "Phoenix Suns": "PHX",
    "Portland Trail Blazers": "POR", "Sacramento Kings": "SAC", "San Antonio Spurs": "SAS",
    "Toronto Raptors": "TOR", "Utah Jazz": "UTA", "Washington Wizards": "WAS",
}

ARENA_COORDS = {
    "ATL": (33.757, -84.396), "BOS": (42.366, -71.062), "BKN": (40.683, -73.976),
    "CHA": (35.225, -80.839), "CHI": (41.881, -87.674), "CLE": (41.496, -81.688),
    "DAL": (32.790, -96.810), "DEN": (39.749, -105.008), "DET": (42.341, -83.055),
    "GSW": (37.768, -122.388), "HOU": (29.751, -95.362), "IND": (39.764, -86.156),
    "LAC": (34.043, -118.267), "LAL": (34.043, -118.267), "MEM": (35.138, -90.051),
    "MIA": (25.781, -80.187), "MIL": (43.045, -87.917), "MIN": (44.980, -93.276),
    "NOP": (29.949, -90.082), "NYK": (40.751, -73.994), "OKC": (35.463, -97.515),
    "ORL": (28.539, -81.384), "PHI": (39.901, -75.172), "PHX": (33.446, -112.071),
    "POR": (45.532, -122.667), "SAC": (38.580, -121.500), "SAS": (29.427, -98.438),
    "TOR": (43.643, -79.379), "UTA": (40.768, -111.901), "WAS": (38.898, -77.021),
}

ARENA_ALTITUDE = {
    "DEN": 5280, "UTA": 4226, "PHX": 1086, "OKC": 1201, "SAS": 650,
    "DAL": 430, "HOU": 43, "MEM": 337, "ATL": 1050, "CHA": 751,
    "IND": 715, "CHI": 594, "MIL": 617, "MIN": 830, "DET": 600,
    "CLE": 653, "BOS": 141, "NYK": 33, "BKN": 33, "PHI": 39,
    "WAS": 25, "MIA": 6, "ORL": 82, "NOP": 7, "TOR": 250,
    "POR": 50, "SAC": 30, "GSW": 12, "LAL": 305, "LAC": 305,
}

TIMEZONE_ET = {
    "ATL": 0, "BOS": 0, "BKN": 0, "CHA": 0, "CHI": -1, "CLE": 0,
    "DAL": -1, "DEN": -2, "DET": 0, "GSW": -3, "HOU": -1, "IND": 0,
    "LAC": -3, "LAL": -3, "MEM": -1, "MIA": 0, "MIL": -1, "MIN": -1,
    "NOP": -1, "NYK": 0, "OKC": -1, "ORL": 0, "PHI": 0, "PHX": -2,
    "POR": -3, "SAC": -3, "SAS": -1, "TOR": 0, "UTA": -2, "WAS": 0,
}

WINDOWS = [3, 5, 7, 10, 15, 20]


def resolve(name):
    if name in TEAM_MAP: return TEAM_MAP[name]
    if len(name) == 3 and name.isupper(): return name
    for full, abbr in TEAM_MAP.items():
        if name in full: return abbr
    return name[:3].upper() if name else None


def haversine(lat1, lon1, lat2, lon2):
    R = 3959
    dlat, dlon = math.radians(lat2 - lat1), math.radians(lon2 - lon1)
    a = math.sin(dlat/2)**2 + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(dlon/2)**2
    return R * 2 * math.asin(math.sqrt(a))


def pull_seasons():
    """Pull NBA game data from nba_api, cache locally."""
    try:
        from nba_api.stats.endpoints import leaguegamefinder
    except ImportError:
        print("[DATA] nba_api not installed, using cached data only")
        return

    existing = {f.stem.replace("games-", "") for f in HIST_DIR.glob("games-*.json")}
    targets = ["2018-19", "2019-20", "2020-21", "2021-22", "2022-23", "2023-24", "2024-25", "2025-26"]
    missing = [s for s in targets if s not in existing]
    if not missing:
        print(f"[DATA] All {len(targets)} seasons cached")
        return

    for season in missing:
        print(f"[DATA] Pulling {season}...")
        try:
            time.sleep(3)
            finder = leaguegamefinder.LeagueGameFinder(
                season_nullable=season, league_id_nullable="00",
                season_type_nullable="Regular Season", timeout=60
            )
            df = finder.get_data_frames()[0]
            if df.empty:
                continue
            pairs = {}
            for _, row in df.iterrows():
                gid = row["GAME_ID"]
                if gid not in pairs:
                    pairs[gid] = []
                pairs[gid].append({
                    "team_name": row.get("TEAM_NAME", ""),
                    "matchup": row.get("MATCHUP", ""),
                    "pts": int(row["PTS"]) if row.get("PTS") is not None else None,
                    "game_date": row.get("GAME_DATE", ""),
                })
            games = []
            for gid, teams in pairs.items():
                if len(teams) != 2:
                    continue
                home = next((t for t in teams if " vs. " in str(t.get("matchup", ""))), None)
                away = next((t for t in teams if " @ " in str(t.get("matchup", ""))), None)
                if not home or not away or home["pts"] is None:
                    continue
                games.append({
                    "game_date": home["game_date"],
                    "home_team": home["team_name"], "away_team": away["team_name"],
                    "home": {"team_name": home["team_name"], "pts": home["pts"]},
                    "away": {"team_name": away["team_name"], "pts": away["pts"]},
                })
            if games:
                (HIST_DIR / f"games-{season}.json").write_text(json.dumps(games))
                print(f"  {len(games)} games saved")
        except Exception as e:
            print(f"  Error pulling {season}: {e}")


def load_all_games():
    """Load all cached game data."""
    games = []
    for f in sorted(HIST_DIR.glob("games-*.json")):
        data = json.loads(f.read_text())
        items = data if isinstance(data, list) else data.get("games", [])
        games.extend(items)
    games.sort(key=lambda g: g.get("game_date", g.get("date", "")))
    return games


# ═══════════════════════════════════════════════════════════
# SECTION 2: FEATURE ENGINE
# ═══════════════════════════════════════════════════════════

FEATURE_ENGINE_VERSION = "genetic-loop-v3"

def build_features(games):
    """Build features from raw game data. Tries real NBAFeatureEngine first, falls back to inline."""
    try:
        from features.engine import NBAFeatureEngine
        engine = NBAFeatureEngine(skip_placeholder=True)
        X, y, feature_names = engine.build(games)
        X = np.nan_to_num(np.array(X, dtype=np.float64), nan=0.0, posinf=1e6, neginf=-1e6)
        y = np.array(y, dtype=np.int32)
        print(f"[ENGINE] Real NBAFeatureEngine: {X.shape[1]} features, {len(y)} games")
        return X, y, feature_names
    except Exception as e:
        print(f"[ENGINE] NBAFeatureEngine import failed ({e}), using inline fallback")
        return _build_features_inline(games)


def _build_features_inline(games):
    """Fallback: Build 250+ inline features from raw game data. Returns X, y, feature_names."""
    team_results = defaultdict(list)
    team_last = {}
    team_elo = defaultdict(lambda: 1500.0)
    X, y = [], []
    feature_names = []
    first = True

    for game in games:
        hr, ar = game.get("home_team", ""), game.get("away_team", "")
        if "home" in game and isinstance(game["home"], dict):
            h, a = game["home"], game.get("away", {})
            hs, as_ = h.get("pts"), a.get("pts")
            if not hr: hr = h.get("team_name", "")
            if not ar: ar = a.get("team_name", "")
        else:
            hs, as_ = game.get("home_score"), game.get("away_score")
        if hs is None or as_ is None:
            continue
        hs, as_ = int(hs), int(as_)
        home, away = resolve(hr), resolve(ar)
        if not home or not away:
            continue
        gd = game.get("game_date", game.get("date", ""))[:10]
        hr_ = team_results[home]
        ar_ = team_results[away]

        if len(hr_) < 5 or len(ar_) < 5:
            team_results[home].append((gd, hs > as_, hs - as_, away, hs, as_))
            team_results[away].append((gd, as_ > hs, as_ - hs, home, as_, hs))
            team_last[home] = gd
            team_last[away] = gd
            K = 20
            exp_h = 1 / (1 + 10 ** ((team_elo[away] - team_elo[home] - 50) / 400))
            team_elo[home] += K * ((1 if hs > as_ else 0) - exp_h)
            team_elo[away] += K * ((0 if hs > as_ else 1) - (1 - exp_h))
            continue

        def wp(r, n):
            s = r[-n:]
            return sum(1 for x in s if x[1]) / len(s) if s else 0.5

        def pd(r, n):
            s = r[-n:]
            return sum(x[2] for x in s) / len(s) if s else 0.0

        def ppg(r, n):
            s = r[-n:]
            return sum(x[4] for x in s) / len(s) if s else 100.0

        def papg(r, n):
            s = r[-n:]
            return sum(x[5] for x in s) / len(s) if s else 100.0

        def strk(r):
            if not r: return 0
            s, l = 0, r[-1][1]
            for x in reversed(r):
                if x[1] == l:
                    s += 1
                else:
                    break
            return s if l else -s

        def close_pct(r, n):
            s = r[-n:]
            return sum(1 for x in s if abs(x[2]) <= 5) / len(s) if s else 0.5

        def blowout_pct(r, n):
            s = r[-n:]
            return sum(1 for x in s if abs(x[2]) >= 15) / len(s) if s else 0.0

        def consistency(r, n):
            s = r[-n:]
            if len(s) < 3: return 0.0
            m = [x[2] for x in s]
            avg = sum(m) / len(m)
            return (sum((v - avg) ** 2 for v in m) / len(m)) ** 0.5

        def rest(t):
            last = team_last.get(t)
            if not last or not gd: return 3
            try:
                return max(0, (datetime.strptime(gd[:10], "%Y-%m-%d") - datetime.strptime(last[:10], "%Y-%m-%d")).days)
            except Exception:
                return 3

        def sos(r, n=10):
            rec = r[-n:]
            if not rec: return 0.5
            ops = [wp(team_results[x[3]], 82) for x in rec if team_results[x[3]]]
            return sum(ops) / len(ops) if ops else 0.5

        def travel_dist(r, team):
            if not r: return 0
            last_opp = r[-1][3]
            if last_opp in ARENA_COORDS and team in ARENA_COORDS:
                return haversine(*ARENA_COORDS[last_opp], *ARENA_COORDS[team])
            return 0

        h_rest, a_rest = rest(home), rest(away)
        try:
            dt = datetime.strptime(gd, "%Y-%m-%d")
            month, dow = dt.month, dt.weekday()
        except Exception:
            month, dow = 1, 2

        sp = max(0, min(1, (month - 10) / 7)) if month >= 10 else max(0, min(1, (month + 2) / 7))

        row = []
        names = []

        # 1. ROLLING PERFORMANCE (96 features)
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for w in WINDOWS:
                row.extend([wp(tr, w), pd(tr, w), ppg(tr, w), papg(tr, w),
                            ppg(tr, w) - papg(tr, w), close_pct(tr, w), blowout_pct(tr, w),
                            ppg(tr, w) + papg(tr, w)])
                if first:
                    names.extend([f"{prefix}_wp{w}", f"{prefix}_pd{w}", f"{prefix}_ppg{w}",
                                  f"{prefix}_papg{w}", f"{prefix}_margin{w}", f"{prefix}_close{w}",
                                  f"{prefix}_blowout{w}", f"{prefix}_ou{w}"])

        # 2. MOMENTUM (16 features)
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            row.extend([strk(tr), abs(strk(tr)),
                        wp(tr, 5) - wp(tr, 82), wp(tr, 3) - wp(tr, 10),
                        ppg(tr, 5) - ppg(tr, 20), papg(tr, 5) - papg(tr, 20),
                        consistency(tr, 10), consistency(tr, 5)])
            if first:
                names.extend([f"{prefix}_streak", f"{prefix}_streak_abs",
                              f"{prefix}_form5v82", f"{prefix}_form3v10",
                              f"{prefix}_scoring_trend", f"{prefix}_defense_trend",
                              f"{prefix}_consistency10", f"{prefix}_consistency5"])

        # 3. REST & SCHEDULE (16 features)
        h_travel = travel_dist(hr_, home)
        a_travel = travel_dist(ar_, away)
        row.extend([
            min(h_rest, 7), min(a_rest, 7), h_rest - a_rest,
            1.0 if h_rest <= 1 else 0.0, 1.0 if a_rest <= 1 else 0.0,
            h_travel / 1000, a_travel / 1000, (h_travel - a_travel) / 1000,
            ARENA_ALTITUDE.get(home, 500) / 5280, ARENA_ALTITUDE.get(away, 500) / 5280,
            (ARENA_ALTITUDE.get(home, 500) - ARENA_ALTITUDE.get(away, 500)) / 5280,
            abs(TIMEZONE_ET.get(home, 0) - TIMEZONE_ET.get(away, 0)),
            0, 0, 0, 0,
        ])
        if first:
            names.extend(["h_rest", "a_rest", "rest_adv", "h_b2b", "a_b2b",
                          "h_travel", "a_travel", "travel_adv",
                          "h_altitude", "a_altitude", "altitude_delta",
                          "tz_shift", "h_games_7d", "a_games_7d", "sched_density", "pad1"])

        # 4. OPPONENT-ADJUSTED (12 features)
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            s5 = sos(tr, 5)
            s10 = sos(tr, 10)
            ss = sos(tr, 82)
            wp_above = sum(1 for r in tr if wp(team_results[r[3]], 82) > 0.5 and r[1]) / max(
                sum(1 for r in tr if wp(team_results[r[3]], 82) > 0.5), 1)
            wp_below = sum(1 for r in tr if wp(team_results[r[3]], 82) <= 0.5 and r[1]) / max(
                sum(1 for r in tr if wp(team_results[r[3]], 82) <= 0.5), 1)
            row.extend([s5, s10, ss, wp_above, wp_below, 0])
            if first:
                names.extend([f"{prefix}_sos5", f"{prefix}_sos10", f"{prefix}_sos_season",
                              f"{prefix}_wp_above500", f"{prefix}_wp_below500", f"{prefix}_margin_quality"])

        # 5. MATCHUP & ELO (12 features)
        row.extend([
            wp(hr_, 10) - wp(ar_, 10), pd(hr_, 10) - pd(ar_, 10),
            ppg(hr_, 10) - papg(ar_, 10), ppg(ar_, 10) - papg(hr_, 10),
            abs(ppg(hr_, 10) + papg(hr_, 10) - ppg(ar_, 10) - papg(ar_, 10)),
            consistency(hr_, 10) - consistency(ar_, 10),
            team_elo[home], team_elo[away], team_elo[home] - team_elo[away] + 50,
            (team_elo[home] - 1500) / 100, (team_elo[away] - 1500) / 100,
            (team_elo[home] - team_elo[away]) / 100,
        ])
        if first:
            names.extend(["rel_strength", "rel_pd", "off_matchup", "def_matchup",
                          "tempo_diff", "consistency_edge",
                          "elo_home", "elo_away", "elo_diff",
                          "elo_home_norm", "elo_away_norm", "elo_diff_norm"])

        # 6. CONTEXT (12 features)
        row.extend([
            1.0, sp, math.sin(2 * math.pi * month / 12), math.cos(2 * math.pi * month / 12),
            dow / 6.0, 1.0 if dow >= 5 else 0.0,
            min(len(hr_), 82) / 82.0, min(len(ar_), 82) / 82.0,
            wp(hr_, 82) + wp(ar_, 82), wp(hr_, 82) - wp(ar_, 82),
            1.0 if wp(hr_, 82) > 0.5 and wp(ar_, 82) > 0.5 else 0.0,
            ppg(hr_, 10) + ppg(ar_, 10),
        ])
        if first:
            names.extend(["home_court", "season_phase", "month_sin", "month_cos",
                          "day_of_week", "is_weekend", "h_games_pct", "a_games_pct",
                          "combined_wp", "wp_diff", "playoff_race", "expected_total"])

        # 7. CROSS-WINDOW MOMENTUM (20 features) — trend acceleration
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            # Short vs long momentum (5 vs 20)
            wp_accel = wp(tr, 3) - 2 * wp(tr, 10) + wp(tr, 20) if len(tr) >= 20 else 0.0
            pd_accel = pd(tr, 3) - 2 * pd(tr, 10) + pd(tr, 20) if len(tr) >= 20 else 0.0
            # Pythagorean expected win rate (Bill James)
            pts_for = sum(x[4] for x in tr[-20:]) if len(tr) >= 5 else 100
            pts_against = sum(x[5] for x in tr[-20:]) if len(tr) >= 5 else 100
            pyth_exp = pts_for ** 13.91 / max(1, pts_for ** 13.91 + pts_against ** 13.91) if pts_for > 0 else 0.5
            # Scoring volatility
            pts_list = [x[4] for x in tr[-10:]] if len(tr) >= 5 else [100]
            pts_vol = (sum((p - sum(pts_list)/len(pts_list))**2 for p in pts_list) / len(pts_list)) ** 0.5 if len(pts_list) > 1 else 0
            # Home/away specific win rates
            home_games = [x for x in tr if x[3] != home] if prefix == "h" else [x for x in tr if x[3] != away]
            ha_wp = sum(1 for x in home_games[-20:] if x[1]) / max(len(home_games[-20:]), 1)
            # Opponent quality of recent wins
            recent_wins = [x for x in tr[-10:] if x[1]]
            win_quality = sum(wp(team_results[x[3]], 82) for x in recent_wins) / max(len(recent_wins), 1) if recent_wins else 0.5
            # Margin trend (linear slope over last 10 games)
            margins_10 = [x[2] for x in tr[-10:]] if len(tr) >= 5 else [0]
            if len(margins_10) >= 3:
                x_vals = list(range(len(margins_10)))
                x_mean = sum(x_vals) / len(x_vals)
                y_mean = sum(margins_10) / len(margins_10)
                num = sum((x - x_mean) * (y - y_mean) for x, y in zip(x_vals, margins_10))
                den = sum((x - x_mean) ** 2 for x in x_vals)
                margin_slope = num / den if den > 0 else 0.0
            else:
                margin_slope = 0.0
            row.extend([
                wp(tr, 5) - wp(tr, 20) if len(tr) >= 20 else 0.0,
                wp_accel, pd_accel, pyth_exp,
                pts_vol / 10.0,  # normalized
                ha_wp, win_quality,
                margin_slope,
                ppg(tr, 3) / max(ppg(tr, 20), 1),  # recent scoring ratio
                papg(tr, 3) / max(papg(tr, 20), 1),  # recent defense ratio
            ])
            if first:
                names.extend([f"{prefix}_wp5v20", f"{prefix}_wp_accel", f"{prefix}_pd_accel",
                              f"{prefix}_pyth_exp", f"{prefix}_pts_vol",
                              f"{prefix}_location_wp", f"{prefix}_win_quality",
                              f"{prefix}_margin_slope", f"{prefix}_off_ratio", f"{prefix}_def_ratio"])

        # 8. INTERACTION FEATURES (12 features) — key cross-terms
        elo_d = team_elo[home] - team_elo[away] + 50
        rest_adv = h_rest - a_rest
        wp_d = wp(hr_, 10) - wp(ar_, 10)
        row.extend([
            elo_d * rest_adv / 10.0,          # elo × rest interaction
            wp_d * rest_adv / 3.0,             # form × rest interaction
            elo_d * (1 if h_rest <= 1 else 0), # elo × b2b penalty
            wp_d ** 2,                          # squared wp diff (nonlinearity)
            elo_d ** 2 / 10000.0,               # squared elo diff
            (ppg(hr_, 10) - papg(ar_, 10)) * (ppg(ar_, 10) - papg(hr_, 10)),  # off×def interaction
            consistency(hr_, 10) * consistency(ar_, 10) / 100.0,  # consistency product
            wp(hr_, 82) * wp(ar_, 82),          # season quality product
            (wp(hr_, 5) - wp(hr_, 20)) * (wp(ar_, 5) - wp(ar_, 20)),  # momentum alignment
            abs(ppg(hr_, 10) + papg(hr_, 10) - ppg(ar_, 10) - papg(ar_, 10)) * elo_d / 1000.0,  # tempo×elo
            (1.0 if wp(hr_, 82) > 0.6 else 0.0) * (1.0 if wp(ar_, 82) < 0.4 else 0.0),  # mismatch flag
            float(h_rest >= 3 and a_rest <= 1),  # rest mismatch flag
        ])
        if first:
            names.extend(["elo_rest_interact", "form_rest_interact", "elo_b2b_penalty",
                          "wp_diff_sq", "elo_diff_sq", "off_def_interact",
                          "consistency_product", "quality_product", "momentum_align",
                          "tempo_elo_interact", "mismatch_flag", "rest_mismatch_flag"])

        # 9. NEW HIGH-IMPACT FEATURES (50 features, windows [5, 10])
        NEW_WINDOWS = [5, 10]

        # Helper: home/away split win% (home team plays at home, away team plays away)
        def home_split_wp(r, n, is_home_team):
            """Win% for home-only or away-only games over last n."""
            if is_home_team:
                # home team's results when they were the home team (opponent is different city)
                loc_games = [x for x in r if x[3] != home][-n:]
            else:
                loc_games = [x for x in r if x[3] != away][-n:]
            if not loc_games:
                return wp(r, n)  # fallback to overall
            return sum(1 for x in loc_games if x[1]) / len(loc_games)

        def away_split_wp(r, n, is_home_team):
            """Win% for away-only games over last n."""
            if is_home_team:
                loc_games = [x for x in r if x[3] == home][-n:]
            else:
                loc_games = [x for x in r if x[3] == away][-n:]
            if not loc_games:
                return wp(r, n)
            return sum(1 for x in loc_games if x[1]) / len(loc_games)

        def net_rating(r, n):
            """Net points per game over window (proxy for net rating)."""
            s = r[-n:]
            if not s:
                return 0.0
            return sum(x[4] - x[5] for x in s) / len(s)

        def pace_proxy(r, n):
            """Approximate pace as total points per game (proxy when possession data absent)."""
            s = r[-n:]
            if not s:
                return 200.0
            return sum(x[4] + x[5] for x in s) / len(s)

        def h2h_wp(hr, ar, n):
            """Head-to-head win% for home team vs this specific away team over last n meetings."""
            meetings = [x for x in hr if x[3] == away][-n:]
            if not meetings:
                return 0.5
            return sum(1 for x in meetings if x[1]) / len(meetings)

        def sos_window(r, n):
            """Average opponent win% over last n games (Strength of Schedule)."""
            rec = r[-n:]
            if not rec:
                return 0.5
            ops = [wp(team_results[x[3]], 82) for x in rec if team_results[x[3]]]
            return sum(ops) / len(ops) if ops else 0.5

        # 9a. Net Rating (windows 5, 10) — 4 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for w in NEW_WINDOWS:
                row.append(net_rating(tr, w))
                if first:
                    names.append(f"{prefix}_net_rating{w}")

        # 9b. Pace proxy (windows 5, 10) — 4 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for w in NEW_WINDOWS:
                row.append(pace_proxy(tr, w))
                if first:
                    names.append(f"{prefix}_pace{w}")

        # 9c. Rest days (already exists as h_rest/a_rest, add explicit named vars for clarity)
        # These are already in section 3 above; skip to avoid duplication.

        # 9d. Home/Away Win% Split (windows 5, 10) — 4 features each side = 8 features
        for w in NEW_WINDOWS:
            row.append(home_split_wp(hr_, w, is_home_team=True))   # h home-venue wp
            row.append(away_split_wp(ar_, w, is_home_team=False))  # a away-venue wp
            if first:
                names.append(f"h_home_wp{w}")
                names.append(f"a_away_wp{w}")

        # 9e. Matchup H2H record (windows 5, 10) — 2 features
        for w in NEW_WINDOWS:
            row.append(h2h_wp(hr_, ar_, w))
            if first:
                names.append(f"h_h2h_wp{w}")

        # 9f. Strength of Schedule windows 5, 10 (distinct from existing sos5/sos10 in sec 4)
        # sec 4 already has h_sos5, h_sos10 — skip to avoid duplication.

        # 9g. Streak type: signed streak (positive=wins, negative=losses) — 2 features
        # (strk() already included in section 2 as h_streak/a_streak; skip duplicate.)

        # 9h. Pace × Net Rating interaction — 4 features (home + away, windows 5 and 10)
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for w in NEW_WINDOWS:
                p = pace_proxy(tr, w)
                n_r = net_rating(tr, w)
                row.append((p * n_r) / 1000.0)  # scaled
                if first:
                    names.append(f"{prefix}_pace_net_interact{w}")

        # 9i. Pythagorean-adjusted net rating (per-100-possessions approximation) — 4 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for w in NEW_WINDOWS:
                s = tr[-w:]
                if s:
                    total_pts_for = sum(x[4] for x in s)
                    total_pts_ag = sum(x[5] for x in s)
                    n_games = len(s)
                    avg_pace = (total_pts_for + total_pts_ag) / max(n_games, 1)
                    # net per 100 possessions approximation
                    net_per100 = ((total_pts_for - total_pts_ag) / max(n_games, 1)) / max(avg_pace / 100.0, 1.0)
                else:
                    net_per100 = 0.0
                row.append(net_per100)
                if first:
                    names.append(f"{prefix}_net_per100_{w}")

        # 9j. Recent opponent quality (win% of opponents faced) — 4 features (windows 5, 10)
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for w in NEW_WINDOWS:
                row.append(sos_window(tr, w))
                if first:
                    names.append(f"{prefix}_opp_quality{w}")

        # ── SECTION 10: EXPONENTIALLY-WEIGHTED MOMENTUM FEATURES (~28 features) ──
        # EWM uses manual exponential decay (no pandas needed) for each team's history.
        # Halflife h means the weight of a game h games ago is 0.5x the weight of the current.
        # alpha = 1 - exp(-ln(2) / halflife)  =>  older games decay exponentially.

        def ewm_win(r, halflife):
            """EWM of wins (0/1) with given halflife in games."""
            s = [x[1] for x in r]
            if not s:
                return 0.5
            alpha = 1.0 - math.exp(-math.log(2) / max(halflife, 0.5))
            val, w_sum = 0.0, 0.0
            for i, v in enumerate(s):
                w = (1 - alpha) ** (len(s) - 1 - i)
                val += w * float(v)
                w_sum += w
            return val / w_sum if w_sum > 0 else 0.5

        def ewm_pd(r, halflife):
            """EWM of point differentials with given halflife."""
            s = [x[2] for x in r]
            if not s:
                return 0.0
            alpha = 1.0 - math.exp(-math.log(2) / max(halflife, 0.5))
            val, w_sum = 0.0, 0.0
            for i, v in enumerate(s):
                w = (1 - alpha) ** (len(s) - 1 - i)
                val += w * v
                w_sum += w
            return val / w_sum if w_sum > 0 else 0.0

        def ewm_ppg(r, halflife):
            """EWM of points scored per game."""
            s = [x[4] for x in r]
            if not s:
                return 100.0
            alpha = 1.0 - math.exp(-math.log(2) / max(halflife, 0.5))
            val, w_sum = 0.0, 0.0
            for i, v in enumerate(s):
                w = (1 - alpha) ** (len(s) - 1 - i)
                val += w * v
                w_sum += w
            return val / w_sum if w_sum > 0 else 100.0

        def ewm_papg(r, halflife):
            """EWM of opponent points per game (defensive rating proxy)."""
            s = [x[5] for x in r]
            if not s:
                return 100.0
            alpha = 1.0 - math.exp(-math.log(2) / max(halflife, 0.5))
            val, w_sum = 0.0, 0.0
            for i, v in enumerate(s):
                w = (1 - alpha) ** (len(s) - 1 - i)
                val += w * v
                w_sum += w
            return val / w_sum if w_sum > 0 else 100.0

        def streak_decay_score(r):
            """current_streak x (1 / (1 + games_since_last_loss)).
            Captures both streak length and recency of last loss."""
            if not r:
                return 0.0
            cur_streak = 0
            last_result = r[-1][1]
            for x in reversed(r):
                if x[1] == last_result:
                    cur_streak += 1
                else:
                    break
            if not last_result:
                return -float(cur_streak)  # losing streak: negative
            # Count consecutive games back since last loss (= win streak length)
            games_since_loss = 0
            for x in reversed(r):
                if not x[1]:
                    break
                games_since_loss += 1
            return cur_streak * (1.0 / (1 + games_since_loss))

        def fatigue_index(r, n=5):
            """Sum of (1/rest_days) for last n inter-game gaps — high = compressed schedule."""
            recent = r[-n:]
            if len(recent) < 2:
                return 0.0
            total = 0.0
            for i in range(1, len(recent)):
                try:
                    d1 = datetime.strptime(recent[i - 1][0][:10], "%Y-%m-%d")
                    d2 = datetime.strptime(recent[i][0][:10], "%Y-%m-%d")
                    gap = max(1, abs((d2 - d1).days))
                    total += 1.0 / gap
                except Exception:
                    total += 0.5  # fallback: assume 2-day gap
            return total

        def b2b_delta(r, metric_idx=2):
            """B2B performance delta: avg metric in B2B games minus avg in normal-rest games.
            B2B = previous game was <= 1 day ago."""
            b2b_vals, normal_vals = [], []
            for i in range(1, len(r)):
                try:
                    d1 = datetime.strptime(r[i - 1][0][:10], "%Y-%m-%d")
                    d2 = datetime.strptime(r[i][0][:10], "%Y-%m-%d")
                    gap = abs((d2 - d1).days)
                except Exception:
                    gap = 2
                val = r[i][metric_idx]
                if gap <= 1:
                    b2b_vals.append(val)
                else:
                    normal_vals.append(val)
            b2b_avg = sum(b2b_vals) / len(b2b_vals) if b2b_vals else 0.0
            normal_avg = sum(normal_vals) / len(normal_vals) if normal_vals else 0.0
            return b2b_avg - normal_avg

        def travel_burden(r, n=7):
            """Count unique opponents (proxy for unique cities visited) in last n games.
            More unique opponents correlates with more travel across the schedule."""
            recent = r[-n:]
            if not recent:
                return 0
            return len({x[3] for x in recent})

        # 10a. EWM Win Probability — halflives [3, 5, 10] x 2 teams = 6 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for hl in [3, 5, 10]:
                row.append(ewm_win(tr, hl))
                if first:
                    names.append(f"{prefix}_ewm_win_hl{hl}")

        # 10b. EWM Point Differential — halflives [3, 5, 10] x 2 teams = 6 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            for hl in [3, 5, 10]:
                row.append(ewm_pd(tr, hl) / 10.0)  # normalize: typical margins ~0–20 pts
                if first:
                    names.append(f"{prefix}_ewm_pd_hl{hl}")

        # 10c. EWM Offensive Rating (halflife=5) — 2 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            row.append(ewm_ppg(tr, 5) / 100.0)  # normalize to ~1.0 range
            if first:
                names.append(f"{prefix}_ewm_off_hl5")

        # 10d. EWM Defensive Rating (halflife=5) — 2 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            row.append(ewm_papg(tr, 5) / 100.0)
            if first:
                names.append(f"{prefix}_ewm_def_hl5")

        # 10e. Streak Decay Score — 2 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            row.append(streak_decay_score(tr))
            if first:
                names.append(f"{prefix}_streak_decay")

        # 10f. Fatigue Index (last 5 games) — 2 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            row.append(fatigue_index(tr, n=5))
            if first:
                names.append(f"{prefix}_fatigue_idx")

        # 10g. B2B Performance Delta (point margin) — 2 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            row.append(b2b_delta(tr, metric_idx=2) / 10.0)  # normalized margin delta
            if first:
                names.append(f"{prefix}_b2b_margin_delta")

        # 10h. Travel Burden (unique cities proxy over last 7 games) — 2 features
        for prefix, tr in [("h", hr_), ("a", ar_)]:
            row.append(float(travel_burden(tr, n=7)) / 7.0)  # normalize to [0, 1]
            if first:
                names.append(f"{prefix}_travel_burden7")

        # 10i. Cross-team EWM interaction features — 4 features
        row.append(ewm_win(hr_, 3) - ewm_win(ar_, 3))         # home vs away momentum (hl=3)
        row.append(ewm_win(hr_, 5) - ewm_win(ar_, 5))         # home vs away momentum (hl=5)
        row.append((ewm_pd(hr_, 5) - ewm_pd(ar_, 5)) / 10.0)  # relative margin quality (hl=5)
        row.append((ewm_ppg(hr_, 5) - ewm_papg(ar_, 5)) / 100.0)  # home offense vs away defense
        if first:
            names.extend(["ewm_win_diff_hl3", "ewm_win_diff_hl5",
                          "ewm_pd_diff_hl5", "ewm_off_vs_def_hl5"])

        X.append(row)
        y.append(1 if hs > as_ else 0)
        if first:
            feature_names = names
            first = False

        team_results[home].append((gd, hs > as_, hs - as_, away, hs, as_))
        team_results[away].append((gd, as_ > hs, as_ - hs, home, as_, hs))
        team_last[home] = gd
        team_last[away] = gd
        K = 20
        exp_h = 1 / (1 + 10 ** ((team_elo[away] - team_elo[home] - 50) / 400))
        team_elo[home] += K * ((1 if hs > as_ else 0) - exp_h)
        team_elo[away] += K * ((0 if hs > as_ else 1) - (1 - exp_h))

    X = np.nan_to_num(np.array(X, dtype=np.float64))
    y = np.array(y, dtype=np.int32)
    return X, y, feature_names


# ═══════════════════════════════════════════════════════════
# SECTION 3: INDIVIDUAL (feature mask + hyperparameters)
# ═══════════════════════════════════════════════════════════

class Individual:
    """One model configuration: feature selection mask + hyperparameters."""

    def __init__(self, n_features, target=100, model_type=None):
        prob = target / max(n_features, 1)
        self.features = [1 if random.random() < prob else 0 for _ in range(n_features)]
        self.hyperparams = {
            "n_estimators": random.randint(100, 600),
            "max_depth": random.randint(3, 10),
            "learning_rate": 10 ** random.uniform(-2.5, -0.5),
            "subsample": random.uniform(0.5, 1.0),
            "colsample_bytree": random.uniform(0.3, 1.0),
            "min_child_weight": random.randint(1, 15),
            "reg_alpha": 10 ** random.uniform(-6, 1),
            "reg_lambda": 10 ** random.uniform(-6, 1),
            "model_type": model_type or random.choice(GPU_MODEL_TYPES),
            # venn_abers added (brain cycle 82, 2026-04-09): HF island best (pareto 0.21773) uses
            # venn_abers; GPU Kaggle loop was missing this option entirely.
            "calibration": random.choices(
                ["none", "sigmoid", "venn_abers", "beta", "isotonic", "isotonic_temporal"],
                weights=[15, 12, 25, 20, 15, 13], k=1)[0],
            # Neural net hyperparams
            "nn_hidden_dims": random.choice([64, 128, 256]),
            "nn_n_layers": random.randint(2, 4),
            "nn_dropout": random.uniform(0.1, 0.5),
            "nn_epochs": random.randint(20, 100),
            "nn_batch_size": random.choice([32, 64, 128]),
        }
        self.fitness = {"brier": 1.0, "roi": 0.0, "sharpe": 0.0, "calibration": 1.0, "calibration_error": 1.0, "composite": 0.0}
        self.pareto_rank = 999
        self.crowding_dist = 0.0
        self.island_id = -1
        self.generation = 0
        self.birth_generation = 0
        self._enforce_feature_cap()

    def selected_indices(self):
        return [i for i, b in enumerate(self.features) if b]

    def to_dict(self):
        return {
            "n_features": self.n_features,
            "hyperparams": {k: v for k, v in self.hyperparams.items()},
            "fitness": dict(self.fitness),
            "generation": self.generation,
        }

    @staticmethod
    def _hamming_distance(f1, f2):
        """Normalized Hamming distance between two binary feature masks (0.0 – 1.0)."""
        n = len(f1)
        if n == 0:
            return 0.0
        return sum(a != b for a, b in zip(f1, f2)) / n

    @staticmethod
    def crossover(p1, p2):
        """Crossover on features + blend hyperparams.

        Crossover type is selected based on parent similarity:
          - Parents very similar (Hamming < 0.1): uniform crossover.
            Picks each bit independently, generating more variation between
            nearly-identical individuals.
          - Otherwise: classic two-point crossover.
        """
        child = Individual.__new__(Individual)
        n = len(p1.features)
        parent_hamming = Individual._hamming_distance(p1.features, p2.features)
        if parent_hamming < 0.1:
            # Uniform crossover: each position drawn independently
            child.features = [
                p1.features[i] if random.random() < 0.5 else p2.features[i]
                for i in range(n)
            ]
        else:
            pt1 = random.randint(0, n - 1)
            pt2 = random.randint(pt1, n - 1)
            child.features = p1.features[:pt1] + p2.features[pt1:pt2] + p1.features[pt2:]

        child.hyperparams = {}
        for key in p1.hyperparams:
            if isinstance(p1.hyperparams[key], (int, float)):
                w = random.random()
                val = w * p1.hyperparams[key] + (1 - w) * p2.hyperparams[key]
                if isinstance(p1.hyperparams[key], int):
                    val = int(round(val))
                child.hyperparams[key] = val
            else:
                child.hyperparams[key] = random.choice([p1.hyperparams[key], p2.hyperparams[key]])

        child.fitness = {"brier": 1.0, "roi": 0.0, "sharpe": 0.0, "calibration": 1.0, "calibration_error": 1.0, "composite": 0.0}
        child.generation = max(p1.generation, p2.generation) + 1
        child.birth_generation = child.generation
        child.pareto_rank = 999
        child.crowding_dist = 0.0
        child.island_id = -1
        child._enforce_feature_cap()
        return child

    MAX_FEATURES = 200  # Hard cap — individuals above this waste compute

    def _enforce_feature_cap(self):
        """If feature count exceeds MAX_FEATURES, randomly drop excess features."""
        selected = [i for i, b in enumerate(self.features) if b]
        if len(selected) > self.MAX_FEATURES:
            to_drop = random.sample(selected, len(selected) - self.MAX_FEATURES)
            for idx in to_drop:
                self.features[idx] = 0
        self.n_features = sum(self.features)

    def mutate(self, rate=0.03):
        """Mutate features and hyperparameters."""
        for i in range(len(self.features)):
            if random.random() < rate:
                self.features[i] = 1 - self.features[i]
        self._enforce_feature_cap()
        if random.random() < 0.15:
            self.hyperparams["n_estimators"] = max(50, self.hyperparams["n_estimators"] + random.randint(-100, 100))
        if random.random() < 0.15:
            self.hyperparams["max_depth"] = max(2, min(12, self.hyperparams["max_depth"] + random.randint(-2, 2)))
        if random.random() < 0.15:
            self.hyperparams["learning_rate"] *= 10 ** random.uniform(-0.3, 0.3)
            self.hyperparams["learning_rate"] = max(0.001, min(0.5, self.hyperparams["learning_rate"]))
        if random.random() < 0.08:
            self.hyperparams["model_type"] = random.choice(GPU_MODEL_TYPES)
        if random.random() < 0.05:
            # Aligned with HF island weights; venn_abers dominant (brain cycle 82).
            self.hyperparams["calibration"] = random.choices(
                ["none", "sigmoid", "venn_abers", "beta", "isotonic", "isotonic_temporal"],
                weights=[16, 12, 25, 20, 14, 13], k=1)[0]
        # Neural net hyperparams
        if random.random() < 0.10:
            self.hyperparams["nn_hidden_dims"] = random.choice([64, 128, 256, 512])
        if random.random() < 0.10:
            self.hyperparams["nn_n_layers"] = max(1, min(6, self.hyperparams.get("nn_n_layers", 2) + random.randint(-1, 1)))
        if random.random() < 0.10:
            self.hyperparams["nn_dropout"] = max(0.0, min(0.7, self.hyperparams.get("nn_dropout", 0.3) + random.uniform(-0.1, 0.1)))


# ═══════════════════════════════════════════════════════════
# SECTION 4: FITNESS EVALUATION (multi-objective)
# ═══════════════════════════════════════════════════════════

def evaluate_individual(ind, X, y, n_splits=5, use_gpu=False, _eval_counter=[0]):
    """
    Evaluate one individual via walk-forward backtest.
    Multi-objective: Brier + ROI + Sharpe + Calibration.
    Includes memory management for 16GB RAM with 500 individuals.

    Post-hoc Platt Scaling (added 2026-03-21):
      Each train fold is split 80/20 into train_proper + calibration_set.
      A LogisticRegression is fitted on (raw_probs_cal, y_cal) and used to
      transform test probabilities → calibrated probabilities before computing
      all downstream metrics (Brier, ROI, ECE).  This removes systematic
      over/under-confidence from tree-based models without touching cv=3 inner
      calibration, giving an expected Brier improvement of -0.008 to -0.015.
    """
    _eval_counter[0] += 1
    if _eval_counter[0] % 10 == 0:
        gc.collect()
    from sklearn.model_selection import TimeSeriesSplit
    from sklearn.metrics import brier_score_loss
    from sklearn.calibration import CalibratedClassifierCV
    from sklearn.linear_model import LogisticRegression

    selected = ind.selected_indices()
    if len(selected) < 15 or len(selected) > Individual.MAX_FEATURES:
        ind.fitness = {"brier": 0.30, "roi": -0.10, "sharpe": -1.0, "calibration": 0.15, "calibration_error": 0.15, "composite": -1.0}
        return

    X_sub = X[:, selected]
    X_sub = np.nan_to_num(X_sub, nan=0.0, posinf=1e6, neginf=-1e6)
    tscv = TimeSeriesSplit(n_splits=n_splits)
    hp = ind.hyperparams

    model = _build_model(hp, use_gpu)
    if model is None:
        ind.fitness["composite"] = -1.0
        return

    is_icl = hp["model_type"] in ICL_MODEL_TYPES
    briers, rois, all_probs, all_y = [], [], [], []

    for ti, vi in tscv.split(X_sub):
        try:
            # ── ICL models (TabICLv2, TabPFN): no clone via get_params, no calibration wrapper ──
            if is_icl:
                m = _build_model(hp, use_gpu)
                m.fit(X_sub[ti], y[ti])
                probs = m.predict_proba(X_sub[vi])[:, 1]
            else:
                # ── Platt Scaling: split train fold 80/20 → proper + calibration ──
                cal_split = max(1, int(len(ti) * 0.20))
                ti_proper = ti[:-cal_split]
                ti_cal    = ti[-cal_split:]

                m = type(model)(**model.get_params())
                if hp["calibration"] != "none":
                    m = CalibratedClassifierCV(m, method=hp["calibration"], cv=3)

                m.fit(X_sub[ti_proper], y[ti_proper])

                raw_cal = m.predict_proba(X_sub[ti_cal])[:, 1].reshape(-1, 1)
                y_cal   = y[ti_cal]

                platt = LogisticRegression(C=1.0, solver="lbfgs", max_iter=200, random_state=42)
                platt.fit(raw_cal, y_cal)

                raw_test = m.predict_proba(X_sub[vi])[:, 1].reshape(-1, 1)
                probs = platt.predict_proba(raw_test)[:, 1]

            briers.append(brier_score_loss(y[vi], probs))
            rois.append(_simulate_betting(probs, y[vi]))
            all_probs.extend(probs)
            all_y.extend(y[vi])
        except Exception:
            briers.append(0.28)
            rois.append(-0.05)

    avg_brier = np.mean(briers)
    avg_roi = np.mean(rois)
    sharpe = np.mean(rois) / max(np.std(rois), 0.01) if len(rois) > 1 else 0.0
    cal_err = _calibration_error(np.array(all_probs), np.array(all_y)) if all_probs else 0.15

    # Multi-objective composite fitness (higher = better)
    # Feature penalty: penalize bloated individuals (n_features > 80)
    n_feat = ind.n_features
    feat_penalty = max(0, (n_feat - 80) / 200) * 0.05  # up to -0.03 for 200 features

    composite = (
        0.40 * (1 - avg_brier) +      # Brier: lower is better
        0.25 * max(0, avg_roi) +        # ROI: higher is better
        0.20 * max(0, sharpe / 3) +     # Sharpe: higher is better
        0.15 * (1 - cal_err)            # Calibration: lower is better
        - feat_penalty                  # Parsimony pressure for n_features > 80
    )

    ind.fitness = {
        "brier": round(avg_brier, 5),
        "roi": round(avg_roi, 4),
        "sharpe": round(sharpe, 4),
        "calibration": round(cal_err, 4),
        "calibration_error": round(cal_err, 4),   # ECE with 10 bins, on calibrated probs
        "composite": round(composite, 5),
    }


def _build_model(hp, use_gpu=False):
    """Build ML model from hyperparameters."""
    mt = hp["model_type"]
    try:
        if mt == "xgboost":
            import xgboost as xgb
            params = {
                "n_estimators": hp["n_estimators"],
                "max_depth": hp["max_depth"],
                "learning_rate": hp["learning_rate"],
                "subsample": hp["subsample"],
                "colsample_bytree": hp["colsample_bytree"],
                "min_child_weight": hp["min_child_weight"],
                "reg_alpha": hp["reg_alpha"],
                "reg_lambda": hp["reg_lambda"],
                "eval_metric": "logloss",
                "random_state": 42,
                "n_jobs": -1,
                "tree_method": "hist",
            }
            if use_gpu:
                params["device"] = "cuda"
            return xgb.XGBClassifier(**params)
        elif mt == "lightgbm":
            import lightgbm as lgbm
            return lgbm.LGBMClassifier(
                n_estimators=hp["n_estimators"],
                max_depth=hp["max_depth"],
                learning_rate=hp["learning_rate"],
                subsample=hp["subsample"],
                num_leaves=min(2 ** hp["max_depth"] - 1, 127),
                reg_alpha=hp["reg_alpha"],
                reg_lambda=hp["reg_lambda"],
                # DART P012: dropout trees — better probability calibration, reduces reliability error
                boosting_type="dart", drop_rate=0.1, skip_drop=0.5, uniform_drop=True,
                verbose=-1, random_state=42, n_jobs=-1,
            )
        elif mt == "catboost":
            from catboost import CatBoostClassifier
            # CPU speed fix: cap iterations to 60 on CPU (catboost is 3-5x slower than lightgbm)
            _cat_iters = hp["n_estimators"]
            if not use_gpu:
                _cat_iters = min(_cat_iters, 60)
            _cat_params = dict(
                iterations=_cat_iters,
                depth=min(hp["max_depth"], 10),
                learning_rate=hp["learning_rate"],
                l2_leaf_reg=hp["reg_lambda"],
                verbose=0, random_state=42,
            )
            if not use_gpu:
                _cat_params["early_stopping_rounds"] = 15
            return CatBoostClassifier(**_cat_params)
        elif mt == "random_forest":
            from sklearn.ensemble import RandomForestClassifier
            return RandomForestClassifier(
                n_estimators=hp["n_estimators"],
                max_depth=hp["max_depth"],
                min_samples_leaf=max(1, hp["min_child_weight"]),
                random_state=42, n_jobs=-1,
            )
        elif mt == "extra_trees":
            from sklearn.ensemble import ExtraTreesClassifier
            return ExtraTreesClassifier(
                n_estimators=hp["n_estimators"],
                max_depth=hp["max_depth"],
                min_samples_leaf=max(1, hp["min_child_weight"]),
                random_state=42, n_jobs=-1,
            )
        elif mt == "xgboost_brier":
            import xgboost as xgb
            def _brier_objective(y_true, y_pred):
                grad = 2.0 * (y_pred - y_true)
                hess = np.full_like(grad, 2.0)
                return grad, hess
            params = {
                "n_estimators": hp["n_estimators"],
                "max_depth": hp["max_depth"],
                "learning_rate": hp["learning_rate"],
                "subsample": hp["subsample"],
                "colsample_bytree": hp["colsample_bytree"],
                "min_child_weight": hp["min_child_weight"],
                "reg_alpha": hp["reg_alpha"],
                "reg_lambda": hp["reg_lambda"],
                "objective": _brier_objective,
                "random_state": 42,
                "n_jobs": -1,
                "tree_method": "hist",
            }
            if use_gpu:
                params["device"] = "cuda"
            return xgb.XGBClassifier(**params)
        elif mt == "tabicl":
            from tabicl import TabICLClassifier
            return TabICLClassifier()
        elif mt == "tabpfn":
            from tabpfn import TabPFNClassifier
            return TabPFNClassifier(device="cuda" if use_gpu else "cpu")
        elif mt == "stacking":
            from sklearn.ensemble import StackingClassifier, RandomForestClassifier, GradientBoostingClassifier
            from sklearn.linear_model import LogisticRegression
            estimators = [
                ("rf", RandomForestClassifier(n_estimators=100, max_depth=hp["max_depth"], random_state=42, n_jobs=-1)),
                ("gb", GradientBoostingClassifier(n_estimators=100, max_depth=min(hp["max_depth"], 6), learning_rate=hp["learning_rate"], random_state=42)),
            ]
            try:
                import xgboost as xgb
                estimators.append(("xgb", xgb.XGBClassifier(n_estimators=100, max_depth=hp["max_depth"], learning_rate=hp["learning_rate"], eval_metric="logloss", random_state=42, n_jobs=-1)))
            except ImportError:
                pass
            return StackingClassifier(estimators=estimators, final_estimator=LogisticRegression(max_iter=500), cv=3, n_jobs=-1)
        elif mt == "mlp":
            from sklearn.neural_network import MLPClassifier
            hidden = tuple([hp.get("nn_hidden_dims", 128)] * hp.get("nn_n_layers", 2))
            return MLPClassifier(
                hidden_layer_sizes=hidden,
                learning_rate_init=hp["learning_rate"],
                max_iter=hp.get("nn_epochs", 50),
                alpha=hp["reg_alpha"],
                random_state=42,
            )
        else:
            # Fallback for unknown types (lstm, transformer, tabnet, etc.) — use GBM
            from sklearn.ensemble import GradientBoostingClassifier
            return GradientBoostingClassifier(
                n_estimators=min(hp["n_estimators"], 200),
                max_depth=hp["max_depth"],
                learning_rate=hp["learning_rate"],
                random_state=42,
            )
    except ImportError:
        from sklearn.ensemble import GradientBoostingClassifier
        return GradientBoostingClassifier(
            n_estimators=min(hp["n_estimators"], 200),
            max_depth=hp["max_depth"],
            learning_rate=hp["learning_rate"],
            random_state=42,
        )
    return None


def _simulate_betting(probs, actuals, edge=0.05, vig=0.045):
    """Simulate flat betting with realistic market odds (including vig).

    Market line estimated as midpoint between our model and 50/50 (conservative).
    Payout at market decimal odds with vig baked in.
    This gives a realistic ROI vs the old fair-value (1/prob) approach.
    """
    stake = 10
    profit = 0
    n_bets = 0
    for prob, actual in zip(probs, actuals):
        # Market prob ~ halfway between our model and 50/50
        market_prob = 0.5 + (prob - 0.5) * 0.5
        if prob > 0.5 + edge:
            # Bet home: market pays at their (less favorable) odds with vig
            market_decimal = 1.0 / (market_prob * (1 + vig / 2))
            n_bets += 1
            if actual == 1:
                profit += stake * (market_decimal - 1)
            else:
                profit -= stake
        elif prob < 0.5 - edge:
            # Bet away
            away_market = 1.0 - market_prob
            market_decimal = 1.0 / (away_market * (1 + vig / 2))
            n_bets += 1
            if actual == 0:
                profit += stake * (market_decimal - 1)
            else:
                profit -= stake
    return profit / (n_bets * stake) if n_bets > 0 else 0.0


def _calibration_error(probs, actuals, n_bins=10):
    """Expected Calibration Error (ECE)."""
    if len(probs) == 0:
        return 1.0
    bins = np.linspace(0, 1, n_bins + 1)
    ece = 0
    for i in range(n_bins):
        mask = (probs >= bins[i]) & (probs < bins[i + 1])
        if mask.sum() == 0:
            continue
        ece += mask.sum() / len(probs) * abs(probs[mask].mean() - actuals[mask].mean())
    return ece


# ═══════════════════════════════════════════════════════════
# SECTION 5: GENETIC EVOLUTION ENGINE
# ═══════════════════════════════════════════════════════════

class GeneticEvolutionEngine:
    """
    REAL genetic evolution engine.
    Runs continuously, evolving a population of model configs.
    """

    def __init__(self, pop_size=500, elite_size=25, mutation_rate=0.15,
                 crossover_rate=0.85, target_features=100, n_splits=3,
                 n_islands=5, migration_interval=10, migrants_per_island=5):
        self.pop_size = pop_size
        self.elite_size = elite_size
        self.base_mutation_rate = mutation_rate
        self.mutation_rate = mutation_rate
        self.mut_floor = 0.05
        self.mut_decay = 0.995
        self.crossover_rate = crossover_rate
        self.target_features = target_features
        self.n_splits = n_splits
        self.n_islands = n_islands
        self.island_size = pop_size // n_islands
        self.migration_interval = migration_interval
        self.migrants_per_island = migrants_per_island

        self.population = []
        self.generation = 0
        self.best_ever = None
        self.history = []
        self.stagnation_counter = 0
        self.use_gpu = False
        # Hamming diversity tracking
        self._pop_centroid = None        # float list — mean feature mask over population
        self._hamming_diversity = 1.0   # normalized average pairwise Hamming distance
        self._no_improve_counter = 0    # gens without best-ever composite improvement

        # Detect GPU
        try:
            import xgboost as xgb
            _test = xgb.XGBClassifier(n_estimators=5, max_depth=3, tree_method="hist", device="cuda")
            _test.fit(np.random.randn(50, 5), np.random.randint(0, 2, 50))
            self.use_gpu = True
            print("[GPU] XGBoost CUDA: ENABLED")
        except Exception:
            print("[GPU] XGBoost CUDA: disabled, using CPU")

    def initialize(self, n_features):
        """Create initial random population."""
        self.n_features = n_features
        self.population = [Individual(n_features, self.target_features) for _ in range(self.pop_size)]
        print(f"[INIT] Population: {self.pop_size} individuals, {n_features} feature candidates, "
              f"~{self.target_features} target features")

    def restore_state(self):
        """Restore population from saved state (survive restarts)."""
        state_file = STATE_DIR / "population.json"
        if not state_file.exists():
            return False
        try:
            state = json.loads(state_file.read_text())
            self.generation = state["generation"]
            self.n_features = state["n_features"]
            self.history = state.get("history", [])
            self.stagnation_counter = state.get("stagnation_counter", 0)
            self.mutation_rate = state.get("mutation_rate", self.base_mutation_rate)

            self.population = []
            for ind_data in state["population"]:
                ind = Individual.__new__(Individual)
                ind.features = ind_data["features"]
                ind.hyperparams = ind_data["hyperparams"]
                ind.fitness = ind_data["fitness"]
                ind.generation = ind_data.get("generation", 0)
                ind.birth_generation = ind_data.get("birth_generation", ind.generation)
                ind.n_features = sum(ind.features)
                self.population.append(ind)

            if state.get("best_ever"):
                be = state["best_ever"]
                self.best_ever = Individual.__new__(Individual)
                self.best_ever.features = be["features"]
                self.best_ever.hyperparams = be["hyperparams"]
                self.best_ever.fitness = be["fitness"]
                self.best_ever.generation = be.get("generation", 0)
                self.best_ever.n_features = sum(self.best_ever.features)

            print(f"[RESTORE] Generation {self.generation}, {len(self.population)} individuals, "
                  f"best Brier={self.best_ever.fitness['brier']:.4f}" if self.best_ever else "")
            return True
        except Exception as e:
            print(f"[RESTORE] Failed: {e}")
            return False

    def resize_population_features(self, new_n_features):
        """Resize feature masks if feature count changed (e.g., new features added)."""
        old_n = self.n_features
        if old_n == new_n_features:
            return
        delta = new_n_features - old_n
        print(f"[RESIZE] Feature count changed: {old_n} -> {new_n_features} (delta={delta})")
        self.n_features = new_n_features
        for ind in self.population:
            if len(ind.features) < new_n_features:
                # Extend with random activation for new features (50% chance each)
                ind.features.extend([1 if random.random() < 0.3 else 0 for _ in range(new_n_features - len(ind.features))])
            elif len(ind.features) > new_n_features:
                ind.features = ind.features[:new_n_features]
            ind.n_features = sum(ind.features)
        if self.best_ever:
            if len(self.best_ever.features) < new_n_features:
                self.best_ever.features.extend([0] * (new_n_features - len(self.best_ever.features)))
            elif len(self.best_ever.features) > new_n_features:
                self.best_ever.features = self.best_ever.features[:new_n_features]
            self.best_ever.n_features = sum(self.best_ever.features)
        print(f"[RESIZE] All {len(self.population)} individuals resized")

    def save_state(self):
        """Save population state to survive restarts."""
        state = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "generation": self.generation,
            "n_features": self.n_features,
            "stagnation_counter": self.stagnation_counter,
            "mutation_rate": self.mutation_rate,
            "population": [
                {
                    "features": ind.features,
                    "hyperparams": {k: (float(v) if isinstance(v, (np.floating,)) else v)
                                    for k, v in ind.hyperparams.items()},
                    "fitness": ind.fitness,
                    "generation": ind.generation,
                    "birth_generation": getattr(ind, 'birth_generation', ind.generation),
                }
                for ind in self.population
            ],
            "best_ever": {
                "features": self.best_ever.features,
                "hyperparams": {k: (float(v) if isinstance(v, (np.floating,)) else v)
                                for k, v in self.best_ever.hyperparams.items()},
                "fitness": self.best_ever.fitness,
                "generation": self.best_ever.generation,
            } if self.best_ever else None,
            "history": self.history[-200:],
        }
        (STATE_DIR / "population.json").write_text(json.dumps(state, default=str))

    # ── Hamming Diversity Utilities ──────────────────────────────────────────

    def _update_pop_centroid(self):
        """Compute and cache the population centroid (mean feature mask).

        The centroid[i] is the fraction of individuals that have feature i active.
        Used by _tournament_select for crowding distance.
        """
        if not self.population:
            return
        n = len(self.population[0].features)
        centroid = [0.0] * n
        for ind in self.population:
            for i, v in enumerate(ind.features):
                centroid[i] += v
        pop_len = len(self.population)
        self._pop_centroid = [c / pop_len for c in centroid]

    def _compute_hamming_diversity(self, sample_size=50):
        """Compute the normalized average pairwise Hamming distance of the population.

        Exact O(N²) computation is expensive for pop_size=500, so we use a
        random sample of up to `sample_size` pairs for efficiency.

        Returns a float in [0, 1].  A value of 0 means all feature masks are
        identical; a value of 1 means every bit differs between every pair.
        """
        pop = self.population
        if len(pop) < 2:
            return 1.0
        n_feat = len(pop[0].features)
        if n_feat == 0:
            return 0.0

        # Random sampling: up to sample_size² / 2 pairs
        indices = list(range(len(pop)))
        random.shuffle(indices)
        sample = indices[:sample_size]

        total_dist = 0.0
        n_pairs = 0
        for i in range(len(sample)):
            for j in range(i + 1, len(sample)):
                f1 = pop[sample[i]].features
                f2 = pop[sample[j]].features
                total_dist += sum(a != b for a, b in zip(f1, f2)) / n_feat
                n_pairs += 1

        return total_dist / n_pairs if n_pairs > 0 else 1.0

    def evolve_one_generation(self, X, y):
        """Run one generation of evolution. Returns best individual."""
        self.generation += 1
        gen_start = time.time()

        # 1. Evaluate all individuals
        for i, ind in enumerate(self.population):
            evaluate_individual(ind, X, y, self.n_splits, self.use_gpu)
            if (i + 1) % 10 == 0:
                print(f"  Evaluated {i+1}/{len(self.population)}...", end="\r")

        # 2. Sort by composite fitness (higher = better)
        self.population.sort(key=lambda x: x.fitness["composite"], reverse=True)
        best = self.population[0]

        # 3. Track best ever
        prev_best_brier = self.best_ever.fitness["brier"] if self.best_ever else 1.0
        if self.best_ever is None or best.fitness["composite"] > self.best_ever.fitness["composite"]:
            self.best_ever = Individual.__new__(Individual)
            self.best_ever.features = best.features[:]
            self.best_ever.hyperparams = dict(best.hyperparams)
            self.best_ever.fitness = dict(best.fitness)
            self.best_ever.n_features = best.n_features
            self.best_ever.generation = self.generation

        # 4. Stagnation detection — track BOTH Brier and composite
        prev_best_composite = self.best_ever.fitness["composite"] if self.best_ever and hasattr(self.best_ever, 'fitness') else 0.0
        brier_stagnant = abs(best.fitness["brier"] - prev_best_brier) < 0.0005
        composite_stagnant = abs(best.fitness["composite"] - prev_best_composite) < 0.001
        if brier_stagnant and composite_stagnant:
            self.stagnation_counter += 1
            self._no_improve_counter += 1
        elif not brier_stagnant:
            self.stagnation_counter = max(0, self.stagnation_counter - 2)  # Partial reset
            self._no_improve_counter = 0
        else:
            self.stagnation_counter = max(0, self.stagnation_counter - 1)
            self._no_improve_counter = 0

        # 4b. Hamming Diversity Monitor
        # Compute normalized average pairwise Hamming distance; also refresh centroid
        # (used by crowding-aware tournament selection below).
        self._hamming_diversity = self._compute_hamming_diversity(sample_size=50)
        self._update_pop_centroid()
        if self._hamming_diversity < 0.15:
            print(f"  [DIVERSITY-LOW] Hamming diversity={self._hamming_diversity:.3f} < 0.15 threshold")

        # 4c. Adaptive Mutation Rate — diversity-driven formula
        # Base = 0.03; rises smoothly toward 0.10 as diversity falls below 0.25.
        # Formula: mutation_rate = 0.03 + 0.07 * max(0, 1 - diversity / 0.25)
        diversity_mutation = 0.03 + 0.07 * max(0.0, 1.0 - self._hamming_diversity / 0.25)
        # Stagnation boosts applied on top (capped at 0.25)
        if self.stagnation_counter >= 10:
            self.mutation_rate = min(0.15, diversity_mutation * 1.8)
            print(f"  [STAGNATION-CRITICAL] {self.stagnation_counter} gens — "
                  f"mutation rate -> {self.mutation_rate:.3f} (diversity={self._hamming_diversity:.3f})")
        elif self.stagnation_counter >= 7:
            self.mutation_rate = min(0.15, diversity_mutation * 1.5)
            print(f"  [STAGNATION] {self.stagnation_counter} gens — "
                  f"mutation rate -> {self.mutation_rate:.3f} (diversity={self._hamming_diversity:.3f})")
        elif self.stagnation_counter >= 3:
            self.mutation_rate = min(0.12, diversity_mutation * 1.2)
        else:
            # Normal regime: formula drives the rate directly
            self.mutation_rate = diversity_mutation

        # 5. Record history
        self.history.append({
            "gen": self.generation,
            "best_brier": best.fitness["brier"],
            "best_roi": best.fitness["roi"],
            "best_sharpe": best.fitness["sharpe"],
            "best_composite": best.fitness["composite"],
            "best_calibration_error": best.fitness.get("calibration_error", best.fitness.get("calibration", 1.0)),
            "n_features": best.n_features,
            "model_type": best.hyperparams["model_type"],
            "mutation_rate": round(self.mutation_rate, 4),
            "avg_composite": round(np.mean([ind.fitness["composite"] for ind in self.population]), 5),
            "pop_diversity": round(np.std([ind.n_features for ind in self.population]), 1),
            "hamming_diversity": round(self._hamming_diversity, 4),
        })

        elapsed = time.time() - gen_start
        ece_val = best.fitness.get("calibration_error", best.fitness.get("calibration", 1.0))
        print(f"  Gen {self.generation}: Brier={best.fitness['brier']:.4f} "
              f"ROI={best.fitness['roi']:.1%} Sharpe={best.fitness['sharpe']:.2f} "
              f"ECE={ece_val:.4f} Features={best.n_features} Model={best.hyperparams['model_type']} "
              f"Composite={best.fitness['composite']:.4f} "
              f"Diversity={self._hamming_diversity:.3f} MutRate={self.mutation_rate:.3f} ({elapsed:.0f}s)")

        # 6. Create next generation
        new_pop = []

        # Elitism — protect top by composite AND top by raw Brier (prevents fossil loss)
        def _clone_individual(src):
            clone = Individual.__new__(Individual)
            clone.features = src.features[:]
            clone.hyperparams = dict(src.hyperparams)
            clone.fitness = dict(src.fitness)
            clone.n_features = src.n_features
            clone.generation = src.generation
            clone.birth_generation = getattr(src, 'birth_generation', src.generation)
            return clone

        # Top elite_size by composite (already sorted)
        elite_ids = set()
        for i in range(min(self.elite_size, len(self.population))):
            new_pop.append(_clone_individual(self.population[i]))
            elite_ids.add(id(self.population[i]))

        # Also protect top-2 by raw Brier score (lower = better) if not already elite
        brier_sorted = sorted(self.population, key=lambda x: x.fitness["brier"])
        for ind in brier_sorted[:2]:
            if id(ind) not in elite_ids:
                new_pop.append(_clone_individual(ind))
                elite_ids.add(id(ind))

        # Aging: remove individuals that have survived > 15 generations without improvement
        MAX_AGE = 15
        aged_out = 0
        for i in range(len(new_pop) - 1, self.elite_size - 1, -1):
            if i < len(new_pop):
                age = self.generation - getattr(new_pop[i], 'birth_generation', 0)
                if age > MAX_AGE and new_pop[i].fitness["composite"] < new_pop[0].fitness["composite"] * 0.95:
                    new_pop.pop(i)
                    aged_out += 1
        if aged_out > 0:
            print(f"  [AGING] {aged_out} stale individuals removed")

        # Injection: smarter — at stagnation >= 7 inject targeted mutants of best, not just random
        n_inject = 0
        if self.stagnation_counter >= 7:
            n_inject = self.pop_size // 4
            # Half random, half targeted mutations of best individual
            n_random = n_inject // 2
            n_mutant = n_inject - n_random
            for _ in range(n_random):
                new_pop.append(Individual(self.n_features, self.target_features))
            # Targeted mutants: take best, apply heavy mutation
            for _ in range(n_mutant):
                mutant = Individual.__new__(Individual)
                mutant.features = self.population[0].features[:]
                mutant.hyperparams = dict(self.population[0].hyperparams)
                mutant.fitness = {"brier": 1.0, "roi": 0.0, "sharpe": 0.0, "calibration": 1.0, "calibration_error": 1.0, "composite": 0.0}
                mutant.birth_generation = self.generation
                mutant.n_features = self.population[0].n_features
                mutant.generation = self.generation
                mutant.mutate(0.25)  # Heavy mutation
                new_pop.append(mutant)
            print(f"  [INJECTION] {n_random} random + {n_mutant} targeted mutants (stagnation={self.stagnation_counter})")
        elif self.stagnation_counter >= 3:
            # Mild injection: 10% fresh individuals
            n_inject = self.pop_size // 10
            for _ in range(n_inject):
                new_pop.append(Individual(self.n_features, self.target_features))
            print(f"  [INJECTION-MILD] {n_inject} fresh individuals (stagnation={self.stagnation_counter})")

        # Diversity Injection: triggered independently when diversity is critically low
        # (diversity < 0.15) OR when there has been no fitness improvement for 5
        # consecutive generations — whichever happens first.  Elites are always kept.
        diversity_trigger = (self._hamming_diversity < 0.15) or (self._no_improve_counter >= 5)
        if diversity_trigger and n_inject == 0:
            # Inject 20% of population as freshly randomized individuals (elites already in new_pop)
            n_diversity_inject = max(1, self.pop_size // 5)
            # Cap to avoid going way over pop_size before the fill loop
            slots_remaining = max(0, self.pop_size - len(new_pop) - n_diversity_inject)
            for _ in range(n_diversity_inject):
                new_pop.append(Individual(self.n_features, self.target_features))
            trigger_reason = (
                f"diversity={self._hamming_diversity:.3f}<0.15"
                if self._hamming_diversity < 0.15
                else f"no_improve={self._no_improve_counter}>=5"
            )
            print(f"  [DIVERSITY-INJECT] {n_diversity_inject} fresh individuals injected "
                  f"({trigger_reason}), elites preserved")

        # Fill with crossover + mutation
        while len(new_pop) < self.pop_size:
            # Diversity-aware tournament: 80% fitness-based, 20% diversity-based
            if random.random() < 0.2:
                p1 = self._diversity_select(7)
                p2 = self._tournament_select(7)
            else:
                p1 = self._tournament_select(7)
                p2 = self._tournament_select(7)
            if random.random() < self.crossover_rate:
                child = Individual.crossover(p1, p2)
            else:
                child = Individual.__new__(Individual)
                child.features = p1.features[:]
                child.hyperparams = dict(p1.hyperparams)
                child.fitness = dict(p1.fitness)
                child.n_features = p1.n_features
                child.generation = self.generation
                child.birth_generation = self.generation
            child.mutate(self.mutation_rate)
            new_pop.append(child)

        self.population = new_pop[:self.pop_size]
        return best

    def _tournament_select(self, k=7):
        """Tournament selection with crowding.

        Standard tournament selection, but when two candidates have similar
        composite fitness (within 5%), prefer the one that is more unique —
        measured by Hamming distance from the population centroid.  This
        implements a lightweight niching pressure that rewards exploration
        without discarding high-quality individuals.
        """
        contestants = random.sample(self.population, min(k, len(self.population)))
        best = max(contestants, key=lambda x: x.fitness["composite"])
        best_fit = best.fitness["composite"]

        # Among contestants within 5% of the best, prefer the most unique one
        similar = [c for c in contestants if best_fit > 0 and
                   abs(c.fitness["composite"] - best_fit) / max(abs(best_fit), 1e-9) < 0.05]
        if len(similar) > 1 and hasattr(self, '_pop_centroid') and self._pop_centroid is not None:
            centroid = self._pop_centroid
            def _dist_from_centroid(ind):
                f = ind.features
                n = len(f)
                if n == 0 or len(centroid) != n:
                    return 0.0
                return sum(abs(f[i] - centroid[i]) for i in range(n)) / n
            best = max(similar, key=_dist_from_centroid)

        return best

    def _diversity_select(self, k=7):
        """Diversity-preserving selection: pick the most unique individual from k random."""
        contestants = random.sample(self.population, min(k, len(self.population)))
        if not self.population:
            return contestants[0]
        # Measure uniqueness: how different is this individual's feature set from the elite?
        elite_features = set()
        for i, ind in enumerate(self.population[:self.elite_size]):
            elite_features.update(ind.selected_indices())
        best_diversity = -1
        best_ind = contestants[0]
        for c in contestants:
            c_features = set(c.selected_indices())
            if not c_features:
                continue
            overlap = len(c_features & elite_features) / max(len(c_features), 1)
            diversity = 1.0 - overlap
            # Weight by fitness to avoid picking terrible individuals
            score = diversity * 0.6 + max(0, c.fitness["composite"]) * 0.4
            if score > best_diversity:
                best_diversity = score
                best_ind = c
        return best_ind

    def save_cycle_results(self, feature_names):
        """Save results after a cycle of generations."""
        if not self.best_ever:
            return

        selected_names = [feature_names[i] for i in self.best_ever.selected_indices()
                          if i < len(feature_names)]

        results = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "generation": self.generation,
            "population_size": self.pop_size,
            "feature_candidates": self.n_features,
            "mutation_rate": round(self.mutation_rate, 4),
            "stagnation_counter": self.stagnation_counter,
            "gpu": self.use_gpu,
            "best": {
                "brier": self.best_ever.fitness["brier"],
                "roi": self.best_ever.fitness["roi"],
                "sharpe": self.best_ever.fitness["sharpe"],
                "calibration": self.best_ever.fitness["calibration"],
                "calibration_error": self.best_ever.fitness.get("calibration_error", self.best_ever.fitness["calibration"]),
                "composite": self.best_ever.fitness["composite"],
                "n_features": self.best_ever.n_features,
                "model_type": self.best_ever.hyperparams["model_type"],
                "hyperparams": {k: (float(v) if isinstance(v, (np.floating, np.integer)) else v)
                                for k, v in self.best_ever.hyperparams.items()},
                "selected_features": selected_names[:50],
            },
            "top5": [ind.to_dict() for ind in sorted(
                self.population, key=lambda x: x.fitness["composite"], reverse=True
            )[:5]],
            "history_last20": self.history[-20:],
        }

        # Save timestamped + latest
        ts = datetime.now().strftime("%Y%m%d-%H%M")
        (RESULTS_DIR / f"evolution-{ts}.json").write_text(json.dumps(results, indent=2, default=str))
        (RESULTS_DIR / "evolution-latest.json").write_text(json.dumps(results, indent=2, default=str))
        return results


# ═══════════════════════════════════════════════════════════
# SECTION 6: VM CALLBACK
# ═══════════════════════════════════════════════════════════

def callback_to_vm(results):
    """POST results to VM data server (best-effort)."""
    import urllib.request
    try:
        url = f"{VM_CALLBACK_URL}/callback/evolution"
        body = json.dumps(results, default=str).encode()
        req = urllib.request.Request(url, data=body, headers={"Content-Type": "application/json"})
        resp = urllib.request.urlopen(req, timeout=10)
        print(f"  [CALLBACK] VM notified: {resp.status}")
    except Exception as e:
        # Best-effort, don't block on failure
        print(f"  [CALLBACK] VM unreachable: {e}")

    # Also try to write to shared mon-ipad data if accessible
    try:
        shared = Path("/home/termius/mon-ipad/data/nba-agent/evolution-latest.json")
        if shared.parent.exists():
            shared.write_text(json.dumps(results, indent=2, default=str))
            print(f"  [CALLBACK] Wrote to mon-ipad")
    except Exception:
        pass


# ═══════════════════════════════════════════════════════════
# SECTION 7: MAIN LOOP (continuous 24/7)
# ═══════════════════════════════════════════════════════════

def run_continuous(generations_per_cycle=10, total_cycles=None, pop_size=500,
                   target_features=100, n_splits=5, cool_down=30):
    """
    Main entry point — runs genetic evolution CONTINUOUSLY.

    Args:
        generations_per_cycle: Generations per cycle before saving/callback
        total_cycles: None = infinite (24/7 mode)
        pop_size: Population size
        target_features: Target number of features per individual
        n_splits: Walk-forward backtest splits
        cool_down: Seconds between cycles
    """
    print("=" * 70)
    print("  NBA QUANT AI — REAL GENETIC EVOLUTION LOOP v3")
    print(f"  Started: {datetime.now(timezone.utc).isoformat()}")
    print(f"  Pop: {pop_size} | Target features: {target_features}")
    print(f"  Gens/cycle: {generations_per_cycle} | Cycles: {'INFINITE' if total_cycles is None else total_cycles}")
    print("=" * 70)

    # 1. Pull data
    print("\n[PHASE 1] Loading data...")
    pull_seasons()
    games = load_all_games()
    print(f"  {len(games)} games loaded")
    if len(games) < 500:
        print("  ERROR: Not enough games!")
        return

    # 2. Build features
    print("\n[PHASE 2] Building features...")
    X, y, feature_names = build_features(games)
    print(f"  Feature matrix: {X.shape} ({len(feature_names)} features)")

    # 3. Initialize engine
    print("\n[PHASE 3] Initializing engine...")
    engine = GeneticEvolutionEngine(
        pop_size=pop_size, elite_size=max(5, pop_size // 20), mutation_rate=0.15,
        crossover_rate=0.85, target_features=target_features, n_splits=n_splits,
        n_islands=5, migration_interval=10, migrants_per_island=5,
    )

    # Try to restore previous state
    if not engine.restore_state():
        engine.initialize(X.shape[1])
    else:
        # Resize population if feature count changed (new features added)
        engine.resize_population_features(X.shape[1])

    # ── Supabase Run Logger + Auto-Cut ──
    run_logger = None
    if _HAS_LOGGER:
        try:
            run_logger = RunLogger(local_dir=str(RESULTS_DIR / "run-logs"))
            print("[RUN-LOGGER] Supabase logging + auto-cut ACTIVE")
        except Exception as e:
            print(f"[RUN-LOGGER] Init failed: {e}")

    # 4. CONTINUOUS EVOLUTION LOOP
    cycle = 0
    while True:
        cycle += 1
        if total_cycles is not None and cycle > total_cycles:
            break

        cycle_start = time.time()
        print(f"\n{'='*60}")
        print(f"  CYCLE {cycle} — Starting {generations_per_cycle} generations")
        print(f"  Time: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}")
        print(f"{'='*60}")

        for gen in range(generations_per_cycle):
            try:
                gen_start = time.time()
                best = engine.evolve_one_generation(X, y)

                # ── Log generation + auto-cut ──
                if run_logger and best:
                    try:
                        pop_div = float(np.std([ind.n_features for ind in engine.population]))
                        avg_comp = float(np.mean([ind.fitness["composite"] for ind in engine.population]))
                        run_logger.log_generation(
                            cycle=cycle, generation=engine.generation,
                            best={"brier": best.fitness["brier"], "roi": best.fitness["roi"],
                                  "sharpe": best.fitness["sharpe"], "composite": best.fitness["composite"],
                                  "n_features": best.n_features, "model_type": best.hyperparams["model_type"]},
                            mutation_rate=engine.mutation_rate, avg_composite=avg_comp,
                            pop_diversity=pop_div, duration_s=time.time() - gen_start)

                        # Auto-cut check
                        cut_actions = run_logger.check_auto_cut(best.fitness, {
                            "mutation_rate": engine.mutation_rate,
                            "stagnation": engine.stagnation_counter,
                            "pop_size": engine.pop_size,
                            "pop_diversity": pop_div,
                        })
                        for action in cut_actions:
                            atype = action["type"]
                            params = action.get("params", {})
                            if atype == "config" and "mutation_rate" in params:
                                engine.mutation_rate = params["mutation_rate"]
                            elif atype == "emergency_diversify":
                                n_new = engine.pop_size // 3
                                engine.population = sorted(engine.population, key=lambda x: x.fitness["composite"], reverse=True)[:engine.pop_size - n_new]
                                for _ in range(n_new):
                                    engine.population.append(Individual(engine.n_features, engine.target_features))
                                print(f"  [AUTO-CUT] Diversified: {n_new} fresh individuals")
                            elif atype == "full_reset":
                                engine.population = sorted(engine.population, key=lambda x: x.fitness["composite"], reverse=True)[:engine.elite_size]
                                while len(engine.population) < engine.pop_size:
                                    engine.population.append(Individual(engine.n_features, engine.target_features))
                                engine.stagnation_counter = 0
                                print(f"  [AUTO-CUT] FULL RESET executed")
                    except Exception as e:
                        print(f"  [RUN-LOGGER] Error: {e}")
            except Exception as e:
                print(f"  [ERROR] Generation failed: {e}")
                traceback.print_exc()
                continue

        # Save state (survives restarts)
        engine.save_state()

        # Save results
        results = engine.save_cycle_results(feature_names)

        cycle_elapsed = time.time() - cycle_start
        print(f"\n  Cycle {cycle} complete in {cycle_elapsed:.0f}s")

        if engine.best_ever:
            print(f"  BEST EVER: Brier={engine.best_ever.fitness['brier']:.4f} "
                  f"ROI={engine.best_ever.fitness['roi']:.1%} "
                  f"Features={engine.best_ever.n_features}")

        # ── Log cycle to Supabase ──
        if run_logger and results and engine.best_ever:
            try:
                pop_div = float(np.std([ind.n_features for ind in engine.population]))
                avg_comp = float(np.mean([ind.fitness["composite"] for ind in engine.population]))
                run_logger.log_cycle(
                    cycle=cycle, generation=engine.generation,
                    best=engine.best_ever.fitness | {"n_features": engine.best_ever.n_features,
                                                      "model_type": engine.best_ever.hyperparams["model_type"]},
                    pop_size=engine.pop_size, mutation_rate=engine.mutation_rate,
                    crossover_rate=engine.crossover_rate, stagnation=engine.stagnation_counter,
                    games=len(games), feature_candidates=X.shape[1],
                    cycle_duration_s=cycle_elapsed, avg_composite=avg_comp, pop_diversity=pop_div,
                    top5=results.get("top5"), selected_features=results.get("best", {}).get("selected_features"))
                print(f"  [RUN-LOGGER] Cycle {cycle} logged to Supabase")
            except Exception as e:
                print(f"  [RUN-LOGGER] Cycle log error: {e}")

        # Callback to VM
        if results:
            callback_to_vm(results)

        # Refresh data periodically (every 10 cycles)
        if cycle % 10 == 0:
            print("\n  [REFRESH] Pulling latest game data...")
            try:
                pull_seasons()
                new_games = load_all_games()
                if len(new_games) > len(games):
                    games = new_games
                    X, y, feature_names = build_features(games)
                    print(f"  [REFRESH] Updated: {X.shape}")
            except Exception as e:
                print(f"  [REFRESH] Failed: {e}")

        if total_cycles is None:
            print(f"\n  Cooling down {cool_down}s before next cycle...")
            time.sleep(cool_down)

    print("\n" + "=" * 70)
    print("  EVOLUTION COMPLETE")
    if engine.best_ever:
        print(f"  Final best: Brier={engine.best_ever.fitness['brier']:.4f} "
              f"ROI={engine.best_ever.fitness['roi']:.1%}")
    print("=" * 70)


# ═══════════════════════════════════════════════════════════
# CLI ENTRY POINT
# ═══════════════════════════════════════════════════════════

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="NBA Quant Genetic Evolution Loop v3")
    parser.add_argument("--continuous", action="store_true", help="Run 24/7 (no cycle limit)")
    parser.add_argument("--generations", type=int, default=10, help="Generations per cycle (default: 10)")
    parser.add_argument("--cycles", type=int, default=None, help="Number of cycles (default: infinite)")
    parser.add_argument("--pop-size", type=int, default=500, help="Population size (default: 500)")
    parser.add_argument("--target-features", type=int, default=100, help="Target features (default: 100)")
    parser.add_argument("--splits", type=int, default=5, help="Walk-forward splits (default: 5)")
    parser.add_argument("--cooldown", type=int, default=30, help="Seconds between cycles (default: 30)")
    args = parser.parse_args()

    cycles = None if args.continuous else (args.cycles or 1)

    run_continuous(
        generations_per_cycle=args.generations,
        total_cycles=cycles,
        pop_size=args.pop_size,
        target_features=args.target_features,
        n_splits=args.splits,
        cool_down=args.cooldown,
    )