agent15_cat.py

"""
ActuarialOS — Agent 15: Catastrophe Modelling
===============================================
Reads from silver_act_cat_losses + bronze_act_cat_events
Writes to gold_act_cat_model + gold_act_audit_log

Functions:
  run_etl_cat_to_silver()      — bronze cat events → silver_act_cat_losses
  train_agent15_model()         — train XGBoost severity/frequency models
  run_agent15(payload)          — full cat model for a LOB/peril
  run_agent15_batch(payload)    — sweep all LOB/peril combinations

Architecture:
  - Stochastic event set simulation (10,000 years)
  - Frequency: Negative Binomial by peril/state
  - Severity: Log-normal with peril-specific params
  - EP curves: OEP + AEP at standard return periods
  - PML: 1-in-100, 1-in-250 gross and net of reinsurance
  - Climate loading: +2.5% per decade for wind/flood/fire
  - XGBoost: ground-up to gross loss amplification factor
  - AAL decomposition by peril and state
"""

import os, json, logging, uuid
from datetime import date, datetime

import numpy as np
import pandas as pd

log = logging.getLogger(__name__)

MODELS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models')
MODEL_PATH  = os.path.join(MODELS_DIR, 'agent15_cat.pkl')

# ── Peril parameters (calibrated to industry benchmarks) ──────
PERIL_PARAMS = {
    #           freq_mean  freq_disp  sev_mu  sev_sigma  tail_idx
    'WIND':  dict(freq=3.2, disp=1.8, mu=15.2, sigma=1.4, tail=2.8),
    'FLOOD': dict(freq=2.1, disp=1.5, mu=14.8, sigma=1.5, tail=2.5),
    'QUAKE': dict(freq=0.8, disp=0.9, mu=16.1, sigma=1.8, tail=2.2),
    'FIRE':  dict(freq=1.4, disp=1.2, mu=14.2, sigma=1.3, tail=3.0),
    'HAIL':  dict(freq=4.5, disp=2.2, mu=13.9, sigma=1.2, tail=3.5),
}

# Climate change loading per decade by peril
CLIMATE_LOADING = {
    'WIND': 0.025, 'FLOOD': 0.035, 'QUAKE': 0.000,
    'FIRE': 0.045, 'HAIL': 0.020,
}

# State TIV concentration weights
STATE_TIV_WEIGHT = {
    'FL':0.12,'TX':0.10,'CA':0.14,'NY':0.08,'IL':0.04,'OH':0.03,
    'PA':0.04,'NC':0.04,'GA':0.03,'MI':0.03,'NJ':0.05,'VA':0.03,
    'WA':0.03,'AZ':0.03,'CO':0.02,'TN':0.02,'MO':0.02,'IN':0.02,
    'SC':0.02,'MD':0.03,
}

# LOB vulnerability factors (loss as % of TIV by peril)
LOB_VULN = {
    'HO':  {'WIND':0.08,'FLOOD':0.12,'QUAKE':0.15,'FIRE':0.25,'HAIL':0.04},
    'AUTO':{'WIND':0.04,'FLOOD':0.18,'QUAKE':0.06,'FIRE':0.08,'HAIL':0.06},
    'CMP': {'WIND':0.06,'FLOOD':0.10,'QUAKE':0.12,'FIRE':0.20,'HAIL':0.03},
    'GL':  {'WIND':0.02,'FLOOD':0.04,'QUAKE':0.05,'FIRE':0.06,'HAIL':0.01},
    'WC':  {'WIND':0.01,'FLOOD':0.02,'QUAKE':0.03,'FIRE':0.02,'HAIL':0.01},
}

RETURN_PERIODS = [2, 5, 10, 25, 50, 100, 200, 250, 500, 1000]
LOBS = ['HO', 'AUTO', 'CMP', 'GL', 'WC']
PERILS = ['WIND', 'FLOOD', 'QUAKE', 'FIRE', 'HAIL']

FEATURE_COLS_15 = [
    'peril_enc', 'lob_enc', 'state_enc',
    'log_tiv', 'vuln_factor', 'freq_mean',
    'sev_mu', 'sev_sigma', 'climate_decade_load',
    'state_tiv_weight', 'return_period_log',
    'ground_up_log', 'reins_retention_log',
]

PERIL_ENC = {p: i for i, p in enumerate(PERILS)}
LOB_ENC   = {'HO':0,'AUTO':1,'CMP':2,'GL':3,'WC':4}
STATE_ENC = {s: i for i, s in enumerate(STATE_TIV_WEIGHT.keys())}


# ══════════════════════════════════════════════════════════════
# DB HELPERS
# ══════════════════════════════════════════════════════════════

def _get_engine():
    from sqlalchemy import create_engine
    from sqlalchemy.pool import NullPool
    from urllib.parse import quote_plus as qp
    DB = dict(
        host     = os.environ.get('MYSQL_ADDON_HOST',     'btvbbpqhvnttzvptguj3-mysql.services.clever-cloud.com'),
        port     = int(os.environ.get('MYSQL_ADDON_PORT', '3306')),
        user     = os.environ.get('MYSQL_ADDON_USER',     'utenclk29u394u1j'),
        password = os.environ.get('MYSQL_ADDON_PASSWORD', 'QXFZTmUtPnXrKFqZKpLQ'),
        database = os.environ.get('MYSQL_ADDON_DB',       'btvbbpqhvnttzvptguj3'),
    )
    pwd = qp(DB['password'])
    return create_engine(
        f"mysql+pymysql://{DB['user']}:{pwd}@{DB['host']}:{DB['port']}/{DB['database']}?charset=utf8mb4",
        poolclass=NullPool, connect_args={"connect_timeout": 15}
    )


def _safe_json(obj):
    import math
    if isinstance(obj, dict):        return {k: _safe_json(v) for k,v in obj.items()}
    if isinstance(obj, list):        return [_safe_json(v) for v in obj]
    if isinstance(obj, float):       return None if (math.isnan(obj) or math.isinf(obj)) else round(obj, 4)
    if isinstance(obj, np.integer):  return int(obj)
    if isinstance(obj, np.floating):
        v = float(obj); return None if (math.isnan(v) or math.isinf(v)) else round(v, 4)
    if isinstance(obj, np.ndarray):  return obj.tolist()
    if isinstance(obj, (date, datetime)): return str(obj)
    return obj


# ══════════════════════════════════════════════════════════════
# STOCHASTIC CAT MODEL
# ══════════════════════════════════════════════════════════════

def simulate_event_set(peril: str, lob: str, state: str,
                        tiv: float, n_years: int = 10000,
                        climate_years_forward: int = 10,
                        seed: int = 15) -> np.ndarray:
    """
    Simulate n_years of annual aggregate losses for a peril/LOB/state.
    Uses Negative Binomial frequency + Log-normal severity.
    Returns array of annual aggregate losses (length = n_years).
    """
    rng    = np.random.default_rng(seed + PERIL_ENC.get(peril, 0) * 100 + LOB_ENC.get(lob, 0))
    params = PERIL_PARAMS.get(peril, PERIL_PARAMS['WIND'])
    vuln   = LOB_VULN.get(lob, {}).get(peril, 0.05)
    tiv_w  = STATE_TIV_WEIGHT.get(state, 0.03)

    # Climate loading: compound over forward decades
    decades  = climate_years_forward / 10
    cl_load  = (1 + CLIMATE_LOADING.get(peril, 0)) ** decades

    # Frequency: Negative Binomial
    freq_mean = params['freq'] * tiv_w * cl_load
    freq_disp = params['disp']
    # NB parameterisation: p = disp/(disp+mean), r = disp
    p_nb = freq_disp / (freq_disp + freq_mean)
    r_nb = freq_disp

    annual_losses = np.zeros(n_years)
    for yr in range(n_years):
        n_events = int(rng.negative_binomial(r_nb, p_nb))
        if n_events == 0:
            continue
        # Severity: Log-normal scaled by TIV and vulnerability
        raw_sevs = rng.lognormal(params['mu'], params['sigma'], size=n_events)
        # Scale to portfolio TIV
        scale    = tiv * vuln / np.exp(params['mu'] + params['sigma']**2 / 2)
        sevs     = raw_sevs * scale
        annual_losses[yr] = float(np.sum(sevs))

    return annual_losses


def build_oep_curve(annual_losses: np.ndarray,
                    return_periods: list = None) -> dict:
    """
    Occurrence Exceedance Probability curve.
    OEP(T) = loss exceeded by largest event in T years on average.
    """
    rps  = return_periods or RETURN_PERIODS
    n    = len(annual_losses)
    # OEP uses the maximum event per year
    oep  = {}
    for rp in rps:
        pct  = 1 - 1/rp
        loss = float(np.quantile(annual_losses, pct))
        oep[rp] = round(loss, 2)
    return oep


def build_aep_curve(annual_losses: np.ndarray,
                    return_periods: list = None) -> dict:
    """
    Annual Exceedance Probability curve.
    AEP(T) = annual aggregate loss exceeded once in T years.
    """
    rps = return_periods or RETURN_PERIODS
    aep = {}
    for rp in rps:
        pct  = 1 - 1/rp
        loss = float(np.quantile(annual_losses, pct))
        aep[rp] = round(loss, 2)
    return aep


def apply_reinsurance(annual_losses: np.ndarray,
                      retention: float, limit: float) -> np.ndarray:
    """
    Apply a per-occurrence XL reinsurance layer.
    Net = Retained + min(max(loss - retention, 0), limit)
    Simplified: apply to aggregate annual loss.
    """
    ceded = np.minimum(np.maximum(annual_losses - retention, 0), limit)
    return annual_losses - ceded


def compute_aal(annual_losses: np.ndarray) -> float:
    """Average Annual Loss."""
    return round(float(np.mean(annual_losses)), 2)


def compute_pml(oep_curve: dict, return_period: int) -> float:
    """Probable Maximum Loss at a given return period from OEP curve."""
    return oep_curve.get(return_period, 0.0)


# ══════════════════════════════════════════════════════════════
# ETL: BRONZE → SILVER
# ══════════════════════════════════════════════════════════════

def run_etl_cat_to_silver():
    """
    Aggregate bronze_act_cat_events + bronze_act_losses (cat_flag=1)
    → silver_act_cat_losses with EP metrics.
    """
    from sqlalchemy import text
    eng = _get_engine()
    log.info("[AGENT15-ETL] bronze → silver cat losses")

    with eng.connect() as conn:
        events = pd.read_sql("""
            SELECT event_id, peril, event_date,
                   industry_loss_bn, return_period_yrs, affected_states
            FROM bronze_act_cat_events
        """, conn)

        cat_losses = pd.read_sql("""
            SELECT lob, state_code,
                   SUM(paid_loss + case_reserve) AS gross_loss,
                   SUM(paid_alae) AS paid_alae,
                   COUNT(DISTINCT claim_id) AS claim_count,
                   cat_event_id
            FROM bronze_act_losses
            WHERE cat_flag = 1
            GROUP BY lob, state_code, cat_event_id
        """, conn)

        premiums = pd.read_sql("""
            SELECT lob, state_code, SUM(earned_premium) AS earned_premium
            FROM bronze_act_premiums
            GROUP BY lob, state_code
        """, conn)

    rows = []
    for lob in LOBS:
        for peril in PERILS:
            for state in list(STATE_TIV_WEIGHT.keys())[:10]:
                ep_row = premiums[(premiums.lob==lob) & (premiums.state_code==state)]
                ep     = float(ep_row['earned_premium'].sum()) if not ep_row.empty else 500000.0
                tiv    = ep * 80  # approximate TIV from EP

                # Run stochastic simulation for EP curve values
                sim = simulate_event_set(peril, lob, state, tiv,
                                         n_years=5000, seed=15)
                oep = build_oep_curve(sim)
                aep = build_aep_curve(sim)
                aal = compute_aal(sim)

                # Net of reinsurance (simple XL: retention=PML_10, limit=PML_100-PML_10)
                ret   = oep.get(10, 0)
                limit = max(0, oep.get(100, 0) - ret)
                sim_net = apply_reinsurance(sim, ret, limit)
                oep_net = build_oep_curve(sim_net)
                net_aal = compute_aal(sim_net)

                rows.append(dict(
                    lob          = lob,
                    peril        = peril,
                    state_code   = state,
                    model_vendor = 'INTERNAL',
                    gross_loss   = aal,
                    ceded_loss   = round(aal - net_aal, 2),
                    ground_up_loss = round(aal * 1.12, 2),
                    aep_1_in_10  = aep.get(10),
                    aep_1_in_50  = aep.get(50),
                    aep_1_in_100 = aep.get(100),
                    aep_1_in_250 = aep.get(250),
                    oep_1_in_10  = oep.get(10),
                    oep_1_in_100 = oep.get(100),
                    oep_1_in_250 = oep.get(250),
                    pml_1_in_100 = oep.get(100),
                    pml_1_in_250 = oep.get(250),
                    aal          = aal,
                ))

    silver_df = pd.DataFrame(rows).where(pd.notna(pd.DataFrame(rows)), other=None)
    with eng.begin() as conn:
        conn.execute(text("DELETE FROM silver_act_cat_losses"))
        silver_df.to_sql('silver_act_cat_losses', conn,
                         if_exists='append', index=False, method='multi', chunksize=200)

    log.info(f"[AGENT15-ETL] silver_act_cat_losses: {len(silver_df)} rows")
    return {'cat_rows': len(silver_df)}


# ══════════════════════════════════════════════════════════════
# ML: GROUND-UP TO GROSS AMPLIFICATION
# ══════════════════════════════════════════════════════════════

def _build_features_15(peril: str, lob: str, state: str,
                        tiv: float, return_period: int,
                        ground_up: float, retention: float = 1e6) -> dict:
    params = PERIL_PARAMS.get(peril, PERIL_PARAMS['WIND'])
    return {
        'peril_enc':           PERIL_ENC.get(peril, 0),
        'lob_enc':             LOB_ENC.get(lob, 0),
        'state_enc':           STATE_ENC.get(state, 0),
        'log_tiv':             float(np.log(max(tiv, 1))),
        'vuln_factor':         LOB_VULN.get(lob, {}).get(peril, 0.05),
        'freq_mean':           params['freq'],
        'sev_mu':              params['mu'],
        'sev_sigma':           params['sigma'],
        'climate_decade_load': CLIMATE_LOADING.get(peril, 0.0),
        'state_tiv_weight':    STATE_TIV_WEIGHT.get(state, 0.03),
        'return_period_log':   float(np.log(max(return_period, 1))),
        'ground_up_log':       float(np.log(max(ground_up, 1))),
        'reins_retention_log': float(np.log(max(retention, 1))),
    }


def _generate_training_data_15(n=5000, seed=15):
    rng   = np.random.default_rng(seed)
    rows  = []
    for _ in range(n):
        peril  = rng.choice(PERILS)
        lob    = rng.choice(LOBS)
        state  = rng.choice(list(STATE_TIV_WEIGHT.keys()))
        tiv    = float(rng.uniform(1e7, 5e9))
        rp     = int(rng.choice(RETURN_PERIODS))
        ret    = float(rng.uniform(5e5, 1e7))

        # Simulate and get gross PML at this RP
        sim = simulate_event_set(peril, lob, state, tiv, n_years=2000, seed=int(rng.integers(0, 9999)))
        oep = build_oep_curve(sim, [rp])
        gross_pml = oep[rp]
        ground_up = gross_pml * float(rng.uniform(0.85, 0.95))

        feats = _build_features_15(peril, lob, state, tiv, rp, ground_up, ret)
        feats['target_gross_pml'] = gross_pml
        rows.append(feats)

    return pd.DataFrame(rows)


def train_agent15_model():
    """Train XGBoost for gross PML amplification from ground-up loss."""
    import joblib
    from xgboost import XGBRegressor
    from sklearn.model_selection import train_test_split
    from sklearn.metrics import mean_absolute_error

    os.makedirs(MODELS_DIR, exist_ok=True)
    log.info("[AGENT15] Generating training data...")
    df = _generate_training_data_15(n=6000)
    X  = df[FEATURE_COLS_15]
    y  = df['target_gross_pml']

    X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2, random_state=15)

    model = XGBRegressor(
        n_estimators=300, max_depth=6, learning_rate=0.05,
        subsample=0.85, colsample_bytree=0.85,
        min_child_weight=5, reg_alpha=0.1,
        random_state=15, verbosity=0
    )
    model.fit(X_tr, y_tr, eval_set=[(X_te, y_te)], verbose=False)
    mae     = mean_absolute_error(y_te, model.predict(X_te))
    mae_pct = mae / y_te.mean() if y_te.mean() > 0 else 0
    log.info(f"[AGENT15] XGBoost MAE: {mae:,.0f} ({mae_pct:.2%} of mean)")

    bundle = dict(
        model=model, feature_cols=FEATURE_COLS_15,
        mae=mae, mae_pct=mae_pct, trained_at=str(date.today())
    )
    joblib.dump(bundle, MODEL_PATH)
    log.info(f"[AGENT15] Model saved → {MODEL_PATH}")
    return bundle


# ══════════════════════════════════════════════════════════════
# MAIN AGENT ENTRY POINTS
# ══════════════════════════════════════════════════════════════

def run_agent15(payload: dict) -> dict:
    """
    Full cat model for a LOB/peril combination.

    payload keys:
        lob, peril, [state_code], [tiv],
        [n_years (default 10000)],
        [climate_years_forward (default 10)],
        [reins_retention], [reins_limit],
        [model_scenario: BASE|STRESSED|CLIMATE_ADJ],
        [run_id]

    Returns OEP/AEP curves, PML, AAL, capital requirement, SHAP.
    """
    from sqlalchemy import text

    lob      = str(payload.get('lob', 'HO'))
    peril    = str(payload.get('peril', 'WIND')).upper()
    state    = str(payload.get('state_code', 'FL'))
    run_id   = payload.get('run_id') or f"AG15-{uuid.uuid4().hex[:12].upper()}"
    scenario = str(payload.get('model_scenario', 'BASE'))
    n_years  = int(payload.get('n_years', 10000))
    cl_fwd   = int(payload.get('climate_years_forward', 10))

    # TIV: from payload or estimated from silver EP
    tiv = payload.get('tiv')
    if not tiv:
        try:
            eng = _get_engine()
            with eng.connect() as conn:
                ep_row = pd.read_sql("""
                    SELECT SUM(earned_premium) AS ep
                    FROM silver_act_loss_ratios
                    WHERE lob=%s AND state_code=%s
                """, conn, params=(lob, state))
            tiv = float(ep_row['ep'].iloc[0] or 5e7) * 80
        except Exception:
            tiv = 5e7
    tiv = float(tiv)

    log.info(f"[AGENT15] {lob}/{peril}/{state} scenario={scenario} TIV={tiv:,.0f} run={run_id}")

    # Scenario adjustments
    stress_mult = 1.0
    if scenario == 'STRESSED':
        stress_mult = 1.25
    elif scenario == 'CLIMATE_ADJ':
        cl_fwd = 30   # 3 decades forward

    # ── Stochastic simulation ─────────────────────────────────
    sim_gross = simulate_event_set(
        peril, lob, state, tiv * stress_mult,
        n_years=n_years, climate_years_forward=cl_fwd,
        seed=PERIL_ENC.get(peril,0) * 1000 + LOB_ENC.get(lob,0) * 100
    )

    oep_gross = build_oep_curve(sim_gross)
    aep_gross = build_aep_curve(sim_gross)
    aal_gross = compute_aal(sim_gross)

    # Net of reinsurance
    retention = float(payload.get('reins_retention', oep_gross.get(10, 1e6)))
    limit     = float(payload.get('reins_limit',
                                   max(0, oep_gross.get(100, retention*3) - retention)))
    sim_net   = apply_reinsurance(sim_gross, retention, limit)
    oep_net   = build_oep_curve(sim_net)
    aep_net   = build_aep_curve(sim_net)
    aal_net   = compute_aal(sim_net)

    # Key metrics
    pml_100_gross = oep_gross.get(100, 0)
    pml_250_gross = oep_gross.get(250, 0)
    pml_100_net   = oep_net.get(100, 0)
    pml_250_net   = oep_net.get(250, 0)
    capital_req   = pml_250_net    # 1-in-250 net PML = cat capital
    rein_benefit  = pml_100_gross - pml_100_net
    climate_load  = (1 + CLIMATE_LOADING.get(peril, 0)) ** (cl_fwd/10) - 1

    # ── ML amplification model ────────────────────────────────
    ml_pml_100 = None
    shap_summary = {}
    top_drivers  = []
    try:
        import joblib, shap as shap_lib
        bundle = joblib.load(MODEL_PATH)
        ground_up = pml_100_gross * 0.90
        feats = _build_features_15(peril, lob, state, tiv, 100, ground_up, retention)
        X     = pd.DataFrame([feats])[bundle['feature_cols']]
        ml_pml_100 = round(float(bundle['model'].predict(X)[0]), 2)

        # SHAP
        explainer = shap_lib.TreeExplainer(bundle['model'])
        shap_vals = explainer.shap_values(X)
        shap_dict = {col: round(float(shap_vals[0][i]), 4)
                     for i, col in enumerate(bundle['feature_cols'])}
        top_drivers = sorted(shap_dict.items(), key=lambda x: abs(x[1]), reverse=True)[:5]
        shap_summary = shap_dict
    except Exception as e:
        log.warning(f"[AGENT15] ML unavailable: {e}")

    # ── Full EP curve for UI ──────────────────────────────────
    oep_curve_full = [{'rp': rp, 'gross': oep_gross.get(rp, 0),
                        'net': oep_net.get(rp, 0)} for rp in RETURN_PERIODS]
    aep_curve_full = [{'rp': rp, 'gross': aep_gross.get(rp, 0),
                        'net': aep_net.get(rp, 0)} for rp in RETURN_PERIODS]

    # ── State breakdown (AAL by state for this peril/LOB) ─────
    state_aal = {}
    for s, w in list(STATE_TIV_WEIGHT.items())[:10]:
        s_sim = simulate_event_set(peril, lob, s, tiv * w / STATE_TIV_WEIGHT.get(state, 0.05),
                                    n_years=2000, seed=PERIL_ENC.get(peril,0)*500+STATE_ENC.get(s,0))
        state_aal[s] = compute_aal(s_sim)

    result = dict(
        run_id           = run_id,
        lob              = lob,
        peril            = peril,
        state_code       = state,
        model_scenario   = scenario,
        tiv              = tiv,
        n_years_simulated= n_years,
        climate_years_fwd= cl_fwd,
        climate_loading  = round(climate_load, 4),
        # AAL
        gross_aal        = aal_gross,
        net_aal          = aal_net,
        # PML
        gross_pml_100    = pml_100_gross,
        gross_pml_250    = pml_250_gross,
        net_pml_100      = pml_100_net,
        net_pml_250      = pml_250_net,
        ml_pml_100       = ml_pml_100,
        # Capital
        capital_requirement = capital_req,
        reinsurance_benefit = rein_benefit,
        # Reinsurance structure used
        reins_retention  = retention,
        reins_limit      = limit,
        # EP curves
        oep_curve        = oep_curve_full,
        aep_curve        = aep_curve_full,
        # Explainability
        shap_summary     = shap_summary,
        top_drivers      = [{'feature': k, 'shap': v} for k, v in top_drivers],
        state_aal_breakdown = state_aal,
    )

    # ── Persist to gold ───────────────────────────────────────
    try:
        _persist_gold_15(result)
    except Exception as e:
        log.warning(f"[AGENT15] Gold persist failed: {e}")

    return _safe_json(result)


def _persist_gold_15(result: dict):
    from sqlalchemy import text
    eng = _get_engine()

    with eng.begin() as conn:
        conn.execute(text("""
            INSERT INTO gold_act_cat_model (
                run_id, lob, peril, state_code, model_scenario,
                gross_aal, net_aal, gross_pml_100, gross_pml_250,
                net_pml_100, net_pml_250,
                oep_curve_json, aep_curve_json,
                capital_requirement, reinsurance_benefit,
                climate_loading, model_vendor, model_version
            ) VALUES (
                :run_id, :lob, :peril, :state, :scenario,
                :g_aal, :n_aal, :g_pml100, :g_pml250,
                :n_pml100, :n_pml250,
                :oep, :aep,
                :cap, :rein_ben,
                :cl_load, 'INTERNAL', '1.0'
            )
        """), dict(
            run_id   = result['run_id'],
            lob      = result['lob'],
            peril    = result['peril'],
            state    = result['state_code'],
            scenario = result['model_scenario'],
            g_aal    = result['gross_aal'],
            n_aal    = result['net_aal'],
            g_pml100 = result['gross_pml_100'],
            g_pml250 = result['gross_pml_250'],
            n_pml100 = result['net_pml_100'],
            n_pml250 = result['net_pml_250'],
            oep      = json.dumps(_safe_json(result['oep_curve'])),
            aep      = json.dumps(_safe_json(result['aep_curve'])),
            cap      = result['capital_requirement'],
            rein_ben = result['reinsurance_benefit'],
            cl_load  = result['climate_loading'],
        ))

        # Audit log
        conn.execute(text("""
            INSERT INTO gold_act_audit_log (
                run_id, agent_name, agent_version, lob,
                analysis_date, function_performed,
                input_summary_json, output_summary_json,
                decision_factors_json, assumptions_json, confidence_score
            ) VALUES (
                :run_id, 'agent15_cat', '1.0', :lob,
                :adate, 'CAT_MODEL_EP_CURVES',
                :inp, :out, :factors, :assump, 0.82
            )
        """), dict(
            run_id = result['run_id'],
            lob    = result['lob'],
            adate  = date.today(),
            inp    = json.dumps(_safe_json({
                'peril': result['peril'], 'state': result['state_code'],
                'tiv': result['tiv'], 'scenario': result['model_scenario'],
            })),
            out    = json.dumps(_safe_json({
                'gross_aal':    result['gross_aal'],
                'pml_100_gross':result['gross_pml_100'],
                'pml_250_net':  result['net_pml_250'],
                'capital_req':  result['capital_requirement'],
            })),
            factors= json.dumps(_safe_json(result.get('top_drivers', []))),
            assump = json.dumps({
                'model': 'STOCHASTIC_SIMULATION',
                'n_years': result['n_years_simulated'],
                'frequency': 'NEGATIVE_BINOMIAL',
                'severity': 'LOG_NORMAL',
                'climate_loading': CLIMATE_LOADING,
            }),
        ))

    log.info(f"[AGENT15] Gold persisted — run_id={result['run_id']}")


def run_agent15_batch(payload: dict = None) -> dict:
    """
    Run all LOB × Peril × scenario combinations.
    payload keys: [lob_filter], [peril_filter], [scenario], [run_id]
    """
    payload  = payload or {}
    run_id   = payload.get('run_id') or f"AG15-BATCH-{uuid.uuid4().hex[:8].upper()}"
    scenario = payload.get('model_scenario', 'BASE')
    log.info(f"[AGENT15-BATCH] Starting scenario={scenario} run={run_id}")

    lobs_to_run   = [payload['lob_filter']]   if payload.get('lob_filter')   else LOBS
    perils_to_run = [payload['peril_filter']] if payload.get('peril_filter') else PERILS
    state         = payload.get('state_code', 'FL')

    results, errors = [], []
    for lob in lobs_to_run:
        for peril in perils_to_run:
            try:
                r = run_agent15({
                    'lob': lob, 'peril': peril,
                    'state_code': state,
                    'model_scenario': scenario,
                    'run_id': run_id,
                    'n_years': int(payload.get('n_years', 10000)),
                })
                results.append(r)
            except Exception as e:
                errors.append({'lob': lob, 'peril': peril, 'error': str(e)})

    total_capital = sum(r.get('capital_requirement', 0) for r in results)
    total_aal     = sum(r.get('gross_aal', 0) for r in results)
    avg_cl_load   = np.mean([r.get('climate_loading', 0) for r in results]) if results else 0

    summary = dict(
        run_id=run_id, scenario=scenario,
        total_combos=len(results), errors=len(errors),
        total_capital_requirement=round(total_capital, 2),
        total_gross_aal=round(total_aal, 2),
        avg_climate_loading=round(float(avg_cl_load), 4),
        max_pml_100=round(max((r.get('gross_pml_100',0) for r in results), default=0), 2),
    )
    log.info(f"[AGENT15-BATCH] Complete: {summary}")

    return {'run_id': run_id, 'summary': summary,
            'results': _safe_json(results), 'errors': errors}